From 8dd51322d1b4382c3a2c1c9d6170ffb6f71172f5 Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 31 Mar 2020 14:24:43 -0700 Subject: [PATCH 01/39] healthcheck: rename healthcheck and associated structs to legacy Signed-off-by: deepthi --- go/cmd/vtcombo/main.go | 6 +- go/cmd/vtgate/status.go | 2 +- go/cmd/vtgate/vtgate.go | 10 +- go/vt/discovery/fake_healthcheck.go | 24 +- .../{healthcheck.go => legacy_healthcheck.go} | 220 +++++++++--------- ...st.go => legacy_healthcheck_flaky_test.go} | 74 +++--- ..._cache.go => legacy_tablet_stats_cache.go} | 84 +++---- ...t.go => legacy_tablet_stats_cache_test.go} | 22 +- ...t.go => legacy_tablet_stats_cache_wait.go} | 10 +- ...=> legacy_tablet_stats_cache_wait_test.go} | 4 +- go/vt/discovery/replicationlag.go | 34 +-- go/vt/discovery/replicationlag_test.go | 50 ++-- go/vt/discovery/tablet_picker.go | 10 +- go/vt/discovery/topology_watcher.go | 2 +- go/vt/discovery/utils.go | 26 ++- go/vt/discovery/utils_test.go | 52 ++--- go/vt/proto/query/query.pb.go | 2 +- go/vt/schemamanager/schemaswap/schema_swap.go | 36 +-- go/vt/throttler/demo/throttler_demo.go | 8 +- go/vt/throttler/max_replication_lag_module.go | 10 +- .../max_replication_lag_module_test.go | 8 +- go/vt/throttler/replication_lag_cache.go | 18 +- go/vt/throttler/replication_lag_record.go | 6 +- go/vt/throttler/throttler.go | 2 +- go/vt/vtctld/realtime_status.go | 4 +- go/vt/vtctld/realtime_status_test.go | 8 +- go/vt/vtctld/tablet_stats_cache.go | 50 ++-- go/vt/vtctld/tablet_stats_cache_test.go | 8 +- go/vt/vtexplain/vtexplain_vtgate.go | 4 +- go/vt/vtgate/api.go | 6 +- go/vt/vtgate/buffer/buffer.go | 8 +- go/vt/vtgate/buffer/buffer_test.go | 28 +-- go/vt/vtgate/discoverygateway.go | 28 ++- go/vt/vtgate/discoverygateway_test.go | 14 +- go/vt/vtgate/executor.go | 2 +- go/vt/vtgate/executor_framework_test.go | 2 +- go/vt/vtgate/gateway.go | 2 +- go/vt/vtgate/grpc_discovery_test.go | 2 +- go/vt/vtgate/scatter_conn.go | 17 +- go/vt/vtgate/scatter_conn_test.go | 4 +- go/vt/vtgate/vstream_manager_test.go | 2 +- go/vt/vtgate/vtgate.go | 170 +++++++------- go/vt/vtgate/vtgate_test.go | 2 +- .../txthrottler/mock_healthcheck_test.go | 10 +- .../txthrottler/mock_throttler_test.go | 2 +- .../tabletserver/txthrottler/tx_throttler.go | 14 +- .../txthrottler/tx_throttler_test.go | 12 +- go/vt/worker/executor.go | 14 +- go/vt/worker/legacy_split_clone.go | 14 +- go/vt/worker/split_clone.go | 20 +- go/vt/worker/tablet_provider.go | 6 +- go/vt/worker/tablet_tracker.go | 2 +- go/vt/worker/tablet_tracker_test.go | 10 +- go/vt/worker/topo_utils.go | 12 +- go/vt/wrangler/keyspace.go | 10 +- 55 files changed, 612 insertions(+), 595 deletions(-) rename go/vt/discovery/{healthcheck.go => legacy_healthcheck.go} (80%) rename go/vt/discovery/{healthcheck_flaky_test.go => legacy_healthcheck_flaky_test.go} (93%) rename go/vt/discovery/{tablet_stats_cache.go => legacy_tablet_stats_cache.go} (73%) rename go/vt/discovery/{tablet_stats_cache_test.go => legacy_tablet_stats_cache_test.go} (94%) rename go/vt/discovery/{tablet_stats_cache_wait.go => legacy_tablet_stats_cache_wait.go} (82%) rename go/vt/discovery/{tablet_stats_cache_wait_test.go => legacy_tablet_stats_cache_wait_test.go} (95%) diff --git a/go/cmd/vtcombo/main.go b/go/cmd/vtcombo/main.go index 4a1d446f3a5..62afaa640bd 100644 --- a/go/cmd/vtcombo/main.go +++ b/go/cmd/vtcombo/main.go @@ -55,7 +55,7 @@ var ( ts *topo.Server resilientServer *srvtopo.ResilientServer - healthCheck discovery.HealthCheck + healthCheck discovery.LegacyHealthCheck ) func init() { @@ -118,7 +118,7 @@ func main() { // vtgate configuration and init resilientServer = srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer") - healthCheck = discovery.NewHealthCheck(1*time.Millisecond /*retryDelay*/, 1*time.Hour /*healthCheckTimeout*/) + healthCheck := discovery.NewLegacyHealthCheck(1*time.Millisecond /*retryDelay*/, 1*time.Hour /*healthCheckTimeout*/) tabletTypesToWait := []topodatapb.TabletType{ topodatapb.TabletType_MASTER, topodatapb.TabletType_REPLICA, @@ -128,7 +128,7 @@ func main() { vtgate.QueryLogHandler = "/debug/vtgate/querylog" vtgate.QueryLogzHandler = "/debug/vtgate/querylogz" vtgate.QueryzHandler = "/debug/vtgate/queryz" - vtg := vtgate.Init(context.Background(), healthCheck, resilientServer, tpb.Cells[0], 2 /*retryCount*/, tabletTypesToWait) + vtg := vtgate.LegacyInit(context.Background(), healthCheck, resilientServer, tpb.Cells[0], 2 /*retryCount*/, tabletTypesToWait) // vtctld configuration and init vtctld.InitVtctld(ts) diff --git a/go/cmd/vtgate/status.go b/go/cmd/vtgate/status.go index f8315945bf8..11d5b93c6fb 100644 --- a/go/cmd/vtgate/status.go +++ b/go/cmd/vtgate/status.go @@ -39,6 +39,6 @@ func addStatusParts(vtg *vtgate.VTGate) { return vtg.GetGatewayCacheStatus() }) servenv.AddStatusPart("Health Check Cache", discovery.HealthCheckTemplate, func() interface{} { - return healthCheck.CacheStatus() + return legacyHealthCheck.CacheStatus() }) } diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index 55779162e35..bbb69e49971 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -45,7 +45,7 @@ var ( ) var resilientServer *srvtopo.ResilientServer -var healthCheck discovery.HealthCheck +var legacyHealthCheck discovery.LegacyHealthCheck func init() { rand.Seed(time.Now().UnixNano()) @@ -63,9 +63,6 @@ func main() { resilientServer = srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer") - healthCheck = discovery.NewHealthCheck(*healthCheckRetryDelay, *healthCheckTimeout) - healthCheck.RegisterStats() - tabletTypes := make([]topodatapb.TabletType, 0, 1) if len(*tabletTypesToWait) != 0 { for _, ttStr := range strings.Split(*tabletTypesToWait, ",") { @@ -78,7 +75,10 @@ func main() { } } - vtg := vtgate.Init(context.Background(), healthCheck, resilientServer, *cell, *retryCount, tabletTypes) + legacyHealthCheck = discovery.NewLegacyHealthCheck(*healthCheckRetryDelay, *healthCheckTimeout) + legacyHealthCheck.RegisterStats() + + vtg := vtgate.LegacyInit(context.Background(), legacyHealthCheck, resilientServer, *cell, *retryCount, tabletTypes) servenv.OnRun(func() { // Flags are parsed now. Parse the template using the actual flag value and overwrite the current template. diff --git a/go/vt/discovery/fake_healthcheck.go b/go/vt/discovery/fake_healthcheck.go index ed3fe2a5996..e19ce218c53 100644 --- a/go/vt/discovery/fake_healthcheck.go +++ b/go/vt/discovery/fake_healthcheck.go @@ -33,7 +33,7 @@ import ( ) // This file contains the definitions for a FakeHealthCheck class to -// simulate a HealthCheck module. Note it is not in a sub-package because +// simulate a LegacyHealthCheck module. Note it is not in a sub-package because // otherwise it couldn't be used in this package's tests because of // circular dependencies. @@ -44,9 +44,9 @@ func NewFakeHealthCheck() *FakeHealthCheck { } } -// FakeHealthCheck implements discovery.HealthCheck. +// FakeHealthCheck implements discovery.LegacyHealthCheck. type FakeHealthCheck struct { - listener HealthCheckStatsListener + listener LegacyHealthCheckStatsListener // mu protects the items map mu sync.RWMutex @@ -54,12 +54,12 @@ type FakeHealthCheck struct { } type fhcItem struct { - ts *TabletStats + ts *LegacyTabletStats conn queryservice.QueryService } // -// discovery.HealthCheck interface methods +// discovery.LegacyHealthCheck interface methods // // RegisterStats is not implemented. @@ -67,7 +67,7 @@ func (fhc *FakeHealthCheck) RegisterStats() { } // SetListener is not implemented. -func (fhc *FakeHealthCheck) SetListener(listener HealthCheckStatsListener, sendDownEvents bool) { +func (fhc *FakeHealthCheck) SetListener(listener LegacyHealthCheckStatsListener, sendDownEvents bool) { fhc.listener = listener } @@ -79,7 +79,7 @@ func (fhc *FakeHealthCheck) WaitForInitialStatsUpdates() { func (fhc *FakeHealthCheck) AddTablet(tablet *topodatapb.Tablet, name string) { key := TabletToMapKey(tablet) item := &fhcItem{ - ts: &TabletStats{ + ts: &LegacyTabletStats{ Key: key, Tablet: tablet, Target: &querypb.Target{ @@ -139,16 +139,16 @@ func (fhc *FakeHealthCheck) GetConnection(key string) queryservice.QueryService } // CacheStatus returns the status for each tablet -func (fhc *FakeHealthCheck) CacheStatus() TabletsCacheStatusList { +func (fhc *FakeHealthCheck) CacheStatus() LegacyTabletsCacheStatusList { fhc.mu.Lock() defer fhc.mu.Unlock() - stats := make(TabletsCacheStatusList, 0, len(fhc.items)) + stats := make(LegacyTabletsCacheStatusList, 0, len(fhc.items)) for _, item := range fhc.items { - stats = append(stats, &TabletsCacheStatus{ + stats = append(stats, &LegacyTabletsCacheStatus{ Cell: "FakeCell", Target: item.ts.Target, - TabletsStats: TabletStatsList{item.ts}, + TabletsStats: LegacyTabletStatsList{item.ts}, }) } sort.Sort(stats) @@ -191,7 +191,7 @@ func (fhc *FakeHealthCheck) AddFakeTablet(cell, host string, port int32, keyspac item := fhc.items[key] if item == nil { item = &fhcItem{ - ts: &TabletStats{ + ts: &LegacyTabletStats{ Key: key, Tablet: t, Up: true, diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/legacy_healthcheck.go similarity index 80% rename from go/vt/discovery/healthcheck.go rename to go/vt/discovery/legacy_healthcheck.go index 9a39e54a0dc..91317ecb931 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/legacy_healthcheck.go @@ -16,23 +16,23 @@ limitations under the License. // Package discovery provides a way to discover all tablets e.g. within a // specific shard and monitor their current health. +// Deprecated +// Use the LegacyHealthCheck object to query for tablets and their health. // -// Use the HealthCheck object to query for tablets and their health. +// For an example how to use the LegacyHealthCheck object, see worker/topo_utils.go. // -// For an example how to use the HealthCheck object, see worker/topo_utils.go. -// -// Tablets have to be manually added to the HealthCheck using AddTablet(). +// Tablets have to be manually added to the LegacyHealthCheck using AddTablet(). // Alternatively, use a Watcher implementation which will constantly watch // a source (e.g. the topology) and add and remove tablets as they are // added or removed from the source. // For a Watcher example have a look at NewShardReplicationWatcher(). // -// Each HealthCheck has a HealthCheckStatsListener that will receive +// Each LegacyHealthCheck has a LegacyHealthCheckStatsListener that will receive // notification of when tablets go up and down. -// TabletStatsCache is one implementation, that caches the known tablets +// LegacyTabletStatsCache is one implementation, that caches the known tablets // and the healthy ones per keyspace/shard/tabletType. // -// Internally, the HealthCheck module is connected to each tablet and has a +// Internally, the LegacyHealthCheck module is connected to each tablet and has a // streaming RPC (StreamHealth) open to receive periodic health infos. package discovery @@ -76,7 +76,7 @@ var ( tabletURLTemplate *template.Template ) -// See the documentation for NewHealthCheck below for an explanation of these parameters. +// See the documentation for NewLegacyHealthCheck below for an explanation of these parameters. const ( DefaultHealthCheckRetryDelay = 5 * time.Second DefaultHealthCheckTimeout = 1 * time.Minute @@ -136,10 +136,10 @@ func ParseTabletURLTemplateFromFlag() { } } -// HealthCheckStatsListener is the listener to receive health check stats update. -type HealthCheckStatsListener interface { +// LegacyHealthCheckStatsListener is the listener to receive health check stats update. +type LegacyHealthCheckStatsListener interface { // StatsUpdate is called when: - // - a new tablet is known to the HealthCheck, and its first + // - a new tablet is known to the LegacyHealthCheck, and its first // streaming healthcheck is returned. (then ts.Up is true). // - a tablet is removed from the list of tablets we watch // (then ts.Up is false). @@ -148,18 +148,18 @@ type HealthCheckStatsListener interface { // (ts.Up false on the old type, ts.Up true on the new type). // If it is false, only one event is sent (ts.Up true on the new // type). - StatsUpdate(*TabletStats) + StatsUpdate(*LegacyTabletStats) } -// TabletStats is returned when getting the set of tablets. -type TabletStats struct { +// LegacyTabletStats is returned when getting the set of tablets. +type LegacyTabletStats struct { // Key uniquely identifies that serving tablet. It is computed // from the Tablet's record Hostname and PortMap. If a tablet // is restarted on different ports, its Key will be different. // Key is computed using the TabletToMapKey method below. // key can be used in GetConnection(). Key string - // Tablet is the tablet object that was sent to HealthCheck.AddTablet. + // Tablet is the tablet object that was sent to LegacyHealthCheck.AddTablet. Tablet *topodatapb.Tablet // Name is an optional tag (e.g. alternative address) for the // tablet. It is supposed to represent the tablet as a task, @@ -186,14 +186,14 @@ type TabletStats struct { LastError error } -// String is defined because we want to print a []*TabletStats array nicely. -func (e *TabletStats) String() string { +// String is defined because we want to print a []*LegacyTabletStats array nicely. +func (e *LegacyTabletStats) String() string { return fmt.Sprint(*e) } -// DeepEqual compares two TabletStats. Since we include protos, we +// DeepEqual compares two LegacyTabletStats. Since we include protos, we // need to use proto.Equal on these. -func (e *TabletStats) DeepEqual(f *TabletStats) bool { +func (e *LegacyTabletStats) DeepEqual(f *LegacyTabletStats) bool { return e.Key == f.Key && proto.Equal(e.Tablet, f.Tablet) && e.Name == f.Name && @@ -206,21 +206,21 @@ func (e *TabletStats) DeepEqual(f *TabletStats) bool { (e.LastError != nil && f.LastError != nil && e.LastError.Error() == f.LastError.Error())) } -// Copy produces a copy of TabletStats. -func (e *TabletStats) Copy() *TabletStats { +// Copy produces a copy of LegacyTabletStats. +func (e *LegacyTabletStats) Copy() *LegacyTabletStats { ts := *e return &ts } // GetTabletHostPort formats a tablet host port address. -func (e TabletStats) GetTabletHostPort() string { +func (e LegacyTabletStats) GetTabletHostPort() string { vtPort := e.Tablet.PortMap["vt"] return netutil.JoinHostPort(e.Tablet.Hostname, vtPort) } // GetHostNameLevel returns the specified hostname level. If the level does not exist it will pick the closest level. // This seems unused but can be utilized by certain url formatting templates. See getTabletDebugURL for more details. -func (e TabletStats) GetHostNameLevel(level int) string { +func (e LegacyTabletStats) GetHostNameLevel(level int) string { chunkedHostname := strings.Split(e.Tablet.Hostname, ".") if level < 0 { @@ -233,15 +233,15 @@ func (e TabletStats) GetHostNameLevel(level int) string { } // NamedStatusURL returns the URL for the case where a tablet server is named. -func (e TabletStats) NamedStatusURL() string { +func (e LegacyTabletStats) NamedStatusURL() string { return "/" + topoproto.TabletAliasString(e.Tablet.Alias) + servenv.StatusURLPath() } // getTabletDebugURL formats a debug url to the tablet. // It uses a format string that can be passed into the app to format // the debug URL to accommodate different network setups. It applies -// the html/template string defined to a TabletStats object. The -// format string can refer to members and functions of TabletStats +// the html/template string defined to a LegacyTabletStats object. The +// format string can refer to members and functions of LegacyTabletStats // like a regular html/template string. // // For instance given a tablet with hostname:port of host.dc.domain:22 @@ -250,13 +250,13 @@ func (e TabletStats) NamedStatusURL() string { // https://{{.Tablet.Hostname}} -> https://host.dc.domain // https://{{.GetHostNameLevel 0}}.bastion.corp -> https://host.bastion.corp // {{.NamedStatusURL}} -> test-0000000001/debug/status -func (e TabletStats) getTabletDebugURL() string { +func (e LegacyTabletStats) getTabletDebugURL() string { var buffer bytes.Buffer tabletURLTemplate.Execute(&buffer, e) return buffer.String() } -// HealthCheck defines the interface of health checking module. +// LegacyHealthCheck defines the interface of health checking module. // The goal of this object is to maintain a StreamHealth RPC // to a lot of tablets. Tablets are added / removed by calling the // AddTablet / RemoveTablet methods (other discovery module objects @@ -266,8 +266,8 @@ func (e TabletStats) getTabletDebugURL() string { // registering a listener. To get the underlying "TabletConn" object // which is used for each tablet, use the "GetConnection()" method // below and pass in the Key string which is also sent to the -// listener in each update (as it is part of TabletStats). -type HealthCheck interface { +// listener in each update (as it is part of LegacyTabletStats). +type LegacyHealthCheck interface { // TabletRecorder interface adds AddTablet and RemoveTablet methods. // AddTablet adds the tablet, and starts health check on it. // RemoveTablet removes the tablet, and stops its StreamHealth RPC. @@ -285,7 +285,7 @@ type HealthCheck interface { // // Note that the default implementation requires to set the // listener before any tablets are added to the healthcheck. - SetListener(listener HealthCheckStatsListener, sendDownEvents bool) + SetListener(listener LegacyHealthCheckStatsListener, sendDownEvents bool) // WaitForInitialStatsUpdates waits until all tablets added via // AddTablet() call were propagated to the listener via corresponding // StatsUpdate() calls. Note that code path from AddTablet() to @@ -298,21 +298,21 @@ type HealthCheck interface { // GetConnection returns the TabletConn of the given tablet. GetConnection(key string) queryservice.QueryService // CacheStatus returns a displayable version of the cache. - CacheStatus() TabletsCacheStatusList + CacheStatus() LegacyTabletsCacheStatusList // Close stops the healthcheck. Close() error } -// HealthCheckImpl performs health checking and notifies downstream components about any changes. -// It contains a map of tabletHealth objects, each of which stores the health information for -// a tablet. A checkConn goroutine is spawned for each tabletHealth, which is responsible for -// keeping that tabletHealth up-to-date. This is done through callbacks to updateHealth. -// If checkConn terminates for any reason, it updates tabletHealth.Up as false. If a tabletHealth +// LegacyHealthCheckImpl performs health checking and notifies downstream components about any changes. +// It contains a map of legacyTabletHealth objects, each of which stores the health information for +// a tablet. A checkConn goroutine is spawned for each legacyTabletHealth, which is responsible for +// keeping that legacyTabletHealth up-to-date. This is done through callbacks to updateHealth. +// If checkConn terminates for any reason, it updates legacyTabletHealth.Up as false. If a legacyTabletHealth // gets removed from the map, its cancelFunc gets called, which ensures that the associated // checkConn goroutine eventually terminates. -type HealthCheckImpl struct { +type LegacyHealthCheckImpl struct { // Immutable fields set at construction time. - listener HealthCheckStatsListener + listener LegacyHealthCheckStatsListener sendDownEvents bool retryDelay time.Duration healthCheckTimeout time.Duration @@ -322,45 +322,45 @@ type HealthCheckImpl struct { // mu protects all the following fields. mu sync.Mutex - // addrToHealth maps from address to tabletHealth. - addrToHealth map[string]*tabletHealth + // addrToHealth maps from address to legacyTabletHealth. + addrToHealth map[string]*legacyTabletHealth // Wait group that's used to wait until all initial StatsUpdate() calls are made after the AddTablet() calls. initialUpdatesWG sync.WaitGroup } -// healthCheckConn is a structure that lives within the scope of +// legacyHealthCheckConn is a structure that lives within the scope of // the checkConn goroutine to maintain its internal state. Therefore, // it does not require synchronization. Changes that are relevant to -// healthcheck are transmitted through calls to HealthCheckImpl.updateHealth. +// healthcheck are transmitted through calls to LegacyHealthCheckImpl.updateHealth. // TODO(sougou): move this and associated functions to a separate file. -type healthCheckConn struct { +type legacyHealthCheckConn struct { ctx context.Context conn queryservice.QueryService - tabletStats TabletStats + tabletStats LegacyTabletStats loggedServingState bool lastResponseTimestamp time.Time // timestamp of the last healthcheck response } -// tabletHealth maintains the health status of a tablet. A map of this -// structure is maintained in HealthCheckImpl. -type tabletHealth struct { - // cancelFunc must be called before discarding tabletHealth. +// legacyTabletHealth maintains the health status of a tablet. A map of this +// structure is maintained in LegacyHealthCheckImpl. +type legacyTabletHealth struct { + // cancelFunc must be called before discarding legacyTabletHealth. // This will ensure that the associated checkConn goroutine will terminate. cancelFunc context.CancelFunc // conn is the connection associated with the tablet. conn queryservice.QueryService // latestTabletStats stores the latest health stats of the tablet. - latestTabletStats TabletStats + latestTabletStats LegacyTabletStats } -// NewDefaultHealthCheck creates a new HealthCheck object with a default configuration. -func NewDefaultHealthCheck() HealthCheck { - return NewHealthCheck(DefaultHealthCheckRetryDelay, DefaultHealthCheckTimeout) +// NewLegacyDefaultHealthCheck creates a new LegacyHealthCheck object with a default configuration. +func NewLegacyDefaultHealthCheck() LegacyHealthCheck { + return NewLegacyHealthCheck(DefaultHealthCheckRetryDelay, DefaultHealthCheckTimeout) } -// NewHealthCheck creates a new HealthCheck object. +// NewLegacyHealthCheck creates a new LegacyHealthCheck object. // Parameters: // retryDelay. // The duration to wait before retrying to connect (e.g. after a failed connection @@ -369,9 +369,9 @@ func NewDefaultHealthCheck() HealthCheck { // The duration for which we consider a health check response to be 'fresh'. If we don't get // a health check response from a tablet for more than this duration, we consider the tablet // not healthy. -func NewHealthCheck(retryDelay, healthCheckTimeout time.Duration) HealthCheck { - hc := &HealthCheckImpl{ - addrToHealth: make(map[string]*tabletHealth), +func NewLegacyHealthCheck(retryDelay, healthCheckTimeout time.Duration) LegacyHealthCheck { + hc := &LegacyHealthCheckImpl{ + addrToHealth: make(map[string]*legacyTabletHealth), retryDelay: retryDelay, healthCheckTimeout: healthCheckTimeout, } @@ -384,7 +384,7 @@ func NewHealthCheck(retryDelay, healthCheckTimeout time.Duration) HealthCheck { } // RegisterStats registers the connection counts stats -func (hc *HealthCheckImpl) RegisterStats() { +func (hc *LegacyHealthCheckImpl) RegisterStats() { stats.NewGaugesFuncWithMultiLabels( "HealthcheckConnections", "the number of healthcheck connections registered", @@ -398,7 +398,7 @@ func (hc *HealthCheckImpl) RegisterStats() { } // ServeHTTP is part of the http.Handler interface. It renders the current state of the discovery gateway tablet cache into json. -func (hc *HealthCheckImpl) ServeHTTP(w http.ResponseWriter, _ *http.Request) { +func (hc *LegacyHealthCheckImpl) ServeHTTP(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json; charset=utf-8") status := hc.cacheStatusMap() b, err := json.MarshalIndent(status, "", " ") @@ -413,7 +413,7 @@ func (hc *HealthCheckImpl) ServeHTTP(w http.ResponseWriter, _ *http.Request) { } // servingConnStats returns the number of serving tablets per keyspace/shard/tablet type. -func (hc *HealthCheckImpl) servingConnStats() map[string]int64 { +func (hc *LegacyHealthCheckImpl) servingConnStats() map[string]int64 { res := make(map[string]int64) hc.mu.Lock() defer hc.mu.Unlock() @@ -428,7 +428,7 @@ func (hc *HealthCheckImpl) servingConnStats() map[string]int64 { } // stateChecksum returns a crc32 checksum of the healthcheck state -func (hc *HealthCheckImpl) stateChecksum() int64 { +func (hc *LegacyHealthCheckImpl) stateChecksum() int64 { // CacheStatus is sorted so this should be stable across vtgates cacheStatus := hc.CacheStatus() var buf bytes.Buffer @@ -449,9 +449,9 @@ func (hc *HealthCheckImpl) stateChecksum() int64 { return int64(crc32.ChecksumIEEE(buf.Bytes())) } -// updateHealth updates the tabletHealth record and transmits the tablet stats +// updateHealth updates the legacyTabletHealth record and transmits the tablet stats // to the listener. -func (hc *HealthCheckImpl) updateHealth(ts *TabletStats, conn queryservice.QueryService) { +func (hc *LegacyHealthCheckImpl) updateHealth(ts *LegacyTabletStats, conn queryservice.QueryService) { // Unconditionally send the received update at the end. defer func() { if hc.listener != nil { @@ -463,7 +463,7 @@ func (hc *HealthCheckImpl) updateHealth(ts *TabletStats, conn queryservice.Query th, ok := hc.addrToHealth[ts.Key] if !ok { // This can happen on delete because the entry is removed first, - // or if HealthCheckImpl has been closed. + // or if LegacyHealthCheckImpl has been closed. hc.mu.Unlock() return } @@ -494,7 +494,7 @@ func (hc *HealthCheckImpl) updateHealth(ts *TabletStats, conn queryservice.Query // finalizeConn closes the health checking connection and sends the final // notification about the tablet to downstream. To be called only on exit from // checkConn(). -func (hc *HealthCheckImpl) finalizeConn(hcc *healthCheckConn) { +func (hc *LegacyHealthCheckImpl) finalizeConn(hcc *legacyHealthCheckConn) { hcc.tabletStats.Up = false hcc.setServingState(false, "finalizeConn closing connection") // Note: checkConn() exits only when hcc.ctx.Done() is closed. Thus it's @@ -512,7 +512,7 @@ func (hc *HealthCheckImpl) finalizeConn(hcc *healthCheckConn) { } // checkConn performs health checking on the given tablet. -func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn, name string) { +func (hc *LegacyHealthCheckImpl) checkConn(hcc *legacyHealthCheckConn, name string) { defer hc.connsWG.Done() defer hc.finalizeConn(hcc) @@ -598,7 +598,7 @@ func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn, name string) { // but don't continue to log if the connection stays down. // // hcc.mu must be locked before calling this function -func (hcc *healthCheckConn) setServingState(serving bool, reason string) { +func (hcc *legacyHealthCheckConn) setServingState(serving bool, reason string) { if !hcc.loggedServingState || (serving != hcc.tabletStats.Serving) { // Emit the log from a separate goroutine to avoid holding // the hcc lock while logging is happening @@ -618,7 +618,7 @@ func (hcc *healthCheckConn) setServingState(serving bool, reason string) { } // stream streams healthcheck responses to callback. -func (hcc *healthCheckConn) stream(ctx context.Context, hc *HealthCheckImpl, callback func(*querypb.StreamHealthResponse) error) { +func (hcc *legacyHealthCheckConn) stream(ctx context.Context, hc *LegacyHealthCheckImpl, callback func(*querypb.StreamHealthResponse) error) { if hcc.conn == nil { conn, err := tabletconn.GetDialer()(hcc.tabletStats.Tablet, grpcclient.FailFast(true)) if err != nil { @@ -640,8 +640,8 @@ func (hcc *healthCheckConn) stream(ctx context.Context, hc *HealthCheckImpl, cal } } -// processResponse reads one health check response, and notifies HealthCheckStatsListener. -func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.StreamHealthResponse) error { +// processResponse reads one health check response, and notifies LegacyHealthCheckStatsListener. +func (hcc *legacyHealthCheckConn) processResponse(hc *LegacyHealthCheckImpl, shr *querypb.StreamHealthResponse) error { select { case <-hcc.ctx.Done(): return hcc.ctx.Err() @@ -661,7 +661,7 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St serving = false } - // hcc.TabletStats.Tablet.Alias.Uid may be 0 because the youtube internal mechanism uses a different + // hcc.LegacyTabletStats.Tablet.Alias.Uid may be 0 because the youtube internal mechanism uses a different // code path to initialize this value. If so, we should skip this check. if shr.TabletAlias != nil && hcc.tabletStats.Tablet.Alias.Uid != 0 && !proto.Equal(shr.TabletAlias, hcc.tabletStats.Tablet.Alias) { return fmt.Errorf("health stats mismatch, tablet %+v alias does not match response alias %v", hcc.tabletStats.Tablet, shr.TabletAlias) @@ -689,7 +689,7 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St return nil } -func (hc *HealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { +func (hc *LegacyHealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { hc.mu.Lock() defer hc.mu.Unlock() @@ -708,16 +708,16 @@ func (hc *HealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { hc.deleteConnLocked(key, th) } -func (hc *HealthCheckImpl) deleteConnLocked(key string, th *tabletHealth) { +func (hc *LegacyHealthCheckImpl) deleteConnLocked(key string, th *legacyTabletHealth) { th.latestTabletStats.Up = false th.cancelFunc() delete(hc.addrToHealth, key) } // SetListener sets the listener for healthcheck updates. -// It must be called after NewHealthCheck and before any tablets are added +// It must be called after NewLegacyHealthCheck and before any tablets are added // (either through AddTablet or through a Watcher). -func (hc *HealthCheckImpl) SetListener(listener HealthCheckStatsListener, sendDownEvents bool) { +func (hc *LegacyHealthCheckImpl) SetListener(listener LegacyHealthCheckStatsListener, sendDownEvents bool) { if hc.listener != nil { panic("must not call SetListener twice") } @@ -735,12 +735,12 @@ func (hc *HealthCheckImpl) SetListener(listener HealthCheckStatsListener, sendDo // AddTablet adds the tablet, and starts health check. // It does not block on making connection. // name is an optional tag for the tablet, e.g. an alternative address. -func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet, name string) { +func (hc *LegacyHealthCheckImpl) AddTablet(tablet *topodatapb.Tablet, name string) { ctx, cancelFunc := context.WithCancel(context.Background()) key := TabletToMapKey(tablet) - hcc := &healthCheckConn{ + hcc := &legacyHealthCheckConn{ ctx: ctx, - tabletStats: TabletStats{ + tabletStats: LegacyTabletStats{ Key: key, Tablet: tablet, Name: name, @@ -767,7 +767,7 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet, name string) { // Remove the old tablet to clear the way. hc.deleteConnLocked(key, th) } - hc.addrToHealth[key] = &tabletHealth{ + hc.addrToHealth[key] = &legacyTabletHealth{ cancelFunc: cancelFunc, latestTabletStats: hcc.tabletStats, } @@ -780,24 +780,24 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet, name string) { // RemoveTablet removes the tablet, and stops the health check. // It does not block. -func (hc *HealthCheckImpl) RemoveTablet(tablet *topodatapb.Tablet) { +func (hc *LegacyHealthCheckImpl) RemoveTablet(tablet *topodatapb.Tablet) { hc.deleteConn(tablet) } // ReplaceTablet removes the old tablet and adds the new tablet. -func (hc *HealthCheckImpl) ReplaceTablet(old, new *topodatapb.Tablet, name string) { +func (hc *LegacyHealthCheckImpl) ReplaceTablet(old, new *topodatapb.Tablet, name string) { hc.deleteConn(old) hc.AddTablet(new, name) } // WaitForInitialStatsUpdates waits until all tablets added via AddTablet() call // were propagated to downstream via corresponding StatsUpdate() calls. -func (hc *HealthCheckImpl) WaitForInitialStatsUpdates() { +func (hc *LegacyHealthCheckImpl) WaitForInitialStatsUpdates() { hc.initialUpdatesWG.Wait() } // GetConnection returns the TabletConn of the given tablet. -func (hc *HealthCheckImpl) GetConnection(key string) queryservice.QueryService { +func (hc *LegacyHealthCheckImpl) GetConnection(key string) queryservice.QueryService { hc.mu.Lock() defer hc.mu.Unlock() @@ -808,23 +808,23 @@ func (hc *HealthCheckImpl) GetConnection(key string) queryservice.QueryService { return th.conn } -// TabletsCacheStatus is the current tablets for a cell/target. -type TabletsCacheStatus struct { +// LegacyTabletsCacheStatus is the current tablets for a cell/target. +type LegacyTabletsCacheStatus struct { Cell string Target *querypb.Target - TabletsStats TabletStatsList + TabletsStats LegacyTabletStatsList } -// TabletStatsList is used for sorting. -type TabletStatsList []*TabletStats +// LegacyTabletStatsList is used for sorting. +type LegacyTabletStatsList []*LegacyTabletStats // Len is part of sort.Interface. -func (tsl TabletStatsList) Len() int { +func (tsl LegacyTabletStatsList) Len() int { return len(tsl) } // Less is part of sort.Interface -func (tsl TabletStatsList) Less(i, j int) bool { +func (tsl LegacyTabletStatsList) Less(i, j int) bool { name1 := tsl[i].Name if name1 == "" { name1 = tsl[i].Key @@ -837,12 +837,12 @@ func (tsl TabletStatsList) Less(i, j int) bool { } // Swap is part of sort.Interface -func (tsl TabletStatsList) Swap(i, j int) { +func (tsl LegacyTabletStatsList) Swap(i, j int) { tsl[i], tsl[j] = tsl[j], tsl[i] } // StatusAsHTML returns an HTML version of the status. -func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML { +func (tcs *LegacyTabletsCacheStatus) StatusAsHTML() template.HTML { tLinks := make([]string, 0, 1) if tcs.TabletsStats != nil { sort.Sort(tcs.TabletsStats) @@ -873,29 +873,29 @@ func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML { return template.HTML(strings.Join(tLinks, "
")) } -// TabletsCacheStatusList is used for sorting. -type TabletsCacheStatusList []*TabletsCacheStatus +// LegacyTabletsCacheStatusList is used for sorting. +type LegacyTabletsCacheStatusList []*LegacyTabletsCacheStatus // Len is part of sort.Interface. -func (tcsl TabletsCacheStatusList) Len() int { +func (tcsl LegacyTabletsCacheStatusList) Len() int { return len(tcsl) } // Less is part of sort.Interface -func (tcsl TabletsCacheStatusList) Less(i, j int) bool { +func (tcsl LegacyTabletsCacheStatusList) Less(i, j int) bool { return tcsl[i].Cell+"."+tcsl[i].Target.Keyspace+"."+tcsl[i].Target.Shard+"."+string(tcsl[i].Target.TabletType) < tcsl[j].Cell+"."+tcsl[j].Target.Keyspace+"."+tcsl[j].Target.Shard+"."+string(tcsl[j].Target.TabletType) } // Swap is part of sort.Interface -func (tcsl TabletsCacheStatusList) Swap(i, j int) { +func (tcsl LegacyTabletsCacheStatusList) Swap(i, j int) { tcsl[i], tcsl[j] = tcsl[j], tcsl[i] } // CacheStatus returns a displayable version of the cache. -func (hc *HealthCheckImpl) CacheStatus() TabletsCacheStatusList { +func (hc *LegacyHealthCheckImpl) CacheStatus() LegacyTabletsCacheStatusList { tcsMap := hc.cacheStatusMap() - tcsl := make(TabletsCacheStatusList, 0, len(tcsMap)) + tcsl := make(LegacyTabletsCacheStatusList, 0, len(tcsMap)) for _, tcs := range tcsMap { tcsl = append(tcsl, tcs) } @@ -903,16 +903,16 @@ func (hc *HealthCheckImpl) CacheStatus() TabletsCacheStatusList { return tcsl } -func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { - tcsMap := make(map[string]*TabletsCacheStatus) +func (hc *LegacyHealthCheckImpl) cacheStatusMap() map[string]*LegacyTabletsCacheStatus { + tcsMap := make(map[string]*LegacyTabletsCacheStatus) hc.mu.Lock() defer hc.mu.Unlock() for _, th := range hc.addrToHealth { key := fmt.Sprintf("%v.%v.%v.%v", th.latestTabletStats.Tablet.Alias.Cell, th.latestTabletStats.Target.Keyspace, th.latestTabletStats.Target.Shard, th.latestTabletStats.Target.TabletType.String()) - var tcs *TabletsCacheStatus + var tcs *LegacyTabletsCacheStatus var ok bool if tcs, ok = tcsMap[key]; !ok { - tcs = &TabletsCacheStatus{ + tcs = &LegacyTabletsCacheStatus{ Cell: th.latestTabletStats.Tablet.Alias.Cell, Target: th.latestTabletStats.Target, } @@ -927,7 +927,7 @@ func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { // Close stops the healthcheck. // After Close() returned, it's guaranteed that the listener isn't // currently executing and won't be called again. -func (hc *HealthCheckImpl) Close() error { +func (hc *LegacyHealthCheckImpl) Close() error { hc.mu.Lock() for _, th := range hc.addrToHealth { th.cancelFunc() @@ -943,15 +943,3 @@ func (hc *HealthCheckImpl) Close() error { return nil } - -// TabletToMapKey creates a key to the map from tablet's host and ports. -// It should only be used in discovery and related module. -func TabletToMapKey(tablet *topodatapb.Tablet) string { - parts := make([]string, 0, 1) - for name, port := range tablet.PortMap { - parts = append(parts, netutil.JoinHostPort(name, port)) - } - sort.Strings(parts) - parts = append([]string{tablet.Hostname}, parts...) - return strings.Join(parts, ",") -} diff --git a/go/vt/discovery/healthcheck_flaky_test.go b/go/vt/discovery/legacy_healthcheck_flaky_test.go similarity index 93% rename from go/vt/discovery/healthcheck_flaky_test.go rename to go/vt/discovery/legacy_healthcheck_flaky_test.go index 58fa12ca0c3..dab8f5bb901 100644 --- a/go/vt/discovery/healthcheck_flaky_test.go +++ b/go/vt/discovery/legacy_healthcheck_flaky_test.go @@ -63,14 +63,14 @@ func TestHealthCheck(t *testing.T) { createFakeConn(tablet, input) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) l := newListener() - hc := NewHealthCheck(1*time.Millisecond, time.Hour).(*HealthCheckImpl) + hc := NewLegacyHealthCheck(1*time.Millisecond, time.Hour).(*LegacyHealthCheckImpl) hc.SetListener(l, true) testChecksum(t, 0, hc.stateChecksum()) hc.AddTablet(tablet, "") - t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + t.Logf(`hc = LegacyHealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. - want := &TabletStats{ + want := &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{}, @@ -90,7 +90,7 @@ func TestHealthCheck(t *testing.T) { TabletExternallyReparentedTimestamp: 10, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - want = &TabletStats{ + want = &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, @@ -112,10 +112,10 @@ func TestHealthCheck(t *testing.T) { } tcsl := hc.CacheStatus() - tcslWant := TabletsCacheStatusList{{ + tcslWant := LegacyTabletsCacheStatusList{{ Cell: "cell", Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, - TabletsStats: TabletStatsList{{ + TabletsStats: LegacyTabletStatsList{{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, @@ -139,7 +139,7 @@ func TestHealthCheck(t *testing.T) { } input <- shr t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.5}}`) - want = &TabletStats{ + want = &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, @@ -152,7 +152,7 @@ func TestHealthCheck(t *testing.T) { if !reflect.DeepEqual(res, want) { t.Errorf(`<-l.output: %+v; want %+v`, res, want) } - want = &TabletStats{ + want = &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -178,7 +178,7 @@ func TestHealthCheck(t *testing.T) { TabletExternallyReparentedTimestamp: 0, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3}, } - want = &TabletStats{ + want = &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -202,7 +202,7 @@ func TestHealthCheck(t *testing.T) { TabletExternallyReparentedTimestamp: 0, RealtimeStats: &querypb.RealtimeStats{HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3}, } - want = &TabletStats{ + want = &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -223,7 +223,7 @@ func TestHealthCheck(t *testing.T) { // remove tablet hc.deleteConn(tablet) t.Logf(`hc.RemoveTablet({Host: "a", PortMap: {"vt": 1}})`) - want = &TabletStats{ + want = &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -251,13 +251,13 @@ func TestHealthCheckStreamError(t *testing.T) { fc.errCh = make(chan error) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) l := newListener() - hc := NewHealthCheck(1*time.Millisecond, time.Hour).(*HealthCheckImpl) + hc := NewLegacyHealthCheck(1*time.Millisecond, time.Hour).(*LegacyHealthCheckImpl) hc.SetListener(l, true) hc.AddTablet(tablet, "") - t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + t.Logf(`hc = LegacyHealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. - want := &TabletStats{ + want := &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{}, @@ -276,7 +276,7 @@ func TestHealthCheckStreamError(t *testing.T) { TabletExternallyReparentedTimestamp: 0, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - want = &TabletStats{ + want = &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -294,7 +294,7 @@ func TestHealthCheckStreamError(t *testing.T) { // Stream error fc.errCh <- fmt.Errorf("some stream error") - want = &TabletStats{ + want = &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -323,13 +323,13 @@ func TestHealthCheckVerifiesTabletAlias(t *testing.T) { t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) l := newListener() - hc := NewHealthCheck(1*time.Millisecond, time.Hour).(*HealthCheckImpl) + hc := NewLegacyHealthCheck(1*time.Millisecond, time.Hour).(*LegacyHealthCheckImpl) hc.SetListener(l, false) hc.AddTablet(tablet, "") - t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + t.Logf(`hc = LegacyHealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. - want := &TabletStats{ + want := &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{}, @@ -389,13 +389,13 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) l := newListener() - hc := NewHealthCheck(1*time.Millisecond, time.Hour).(*HealthCheckImpl) + hc := NewLegacyHealthCheck(1*time.Millisecond, time.Hour).(*LegacyHealthCheckImpl) hc.SetListener(l, false) hc.AddTablet(tablet, "") - t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + t.Logf(`hc = LegacyHealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. - want := &TabletStats{ + want := &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{}, @@ -414,7 +414,7 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { TabletExternallyReparentedTimestamp: 10, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - want = &TabletStats{ + want = &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, @@ -452,7 +452,7 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { select { case res = <-l.output: if res.TabletExternallyReparentedTimestamp == 10 && res.LastError == context.Canceled { - // HealthCheck repeats the previous stats if there is an error. + // LegacyHealthCheck repeats the previous stats if there is an error. // This is expected. break } @@ -484,13 +484,13 @@ func TestHealthCheckTimeout(t *testing.T) { fc := createFakeConn(tablet, input) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) l := newListener() - hc := NewHealthCheck(1*time.Millisecond, timeout).(*HealthCheckImpl) + hc := NewLegacyHealthCheck(1*time.Millisecond, timeout).(*LegacyHealthCheckImpl) hc.SetListener(l, false) hc.AddTablet(tablet, "") - t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + t.Logf(`hc = LegacyHealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. - want := &TabletStats{ + want := &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{}, @@ -509,7 +509,7 @@ func TestHealthCheckTimeout(t *testing.T) { TabletExternallyReparentedTimestamp: 10, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - want = &TabletStats{ + want = &LegacyTabletStats{ Key: "a,vt:1", Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, @@ -581,7 +581,7 @@ func TestHealthCheckTimeout(t *testing.T) { func TestTemplate(t *testing.T) { tablet := topo.NewTablet(0, "cell", "a") - ts := []*TabletStats{ + ts := []*LegacyTabletStats{ { Key: "a", Tablet: tablet, @@ -592,7 +592,7 @@ func TestTemplate(t *testing.T) { TabletExternallyReparentedTimestamp: 0, }, } - tcs := &TabletsCacheStatus{ + tcs := &LegacyTabletsCacheStatus{ Cell: "cell", Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, TabletsStats: ts, @@ -603,7 +603,7 @@ func TestTemplate(t *testing.T) { t.Fatalf("error parsing template: %v", err) } wr := &bytes.Buffer{} - if err := templ.Execute(wr, []*TabletsCacheStatus{tcs}); err != nil { + if err := templ.Execute(wr, []*LegacyTabletsCacheStatus{tcs}); err != nil { t.Fatalf("error executing template: %v", err) } } @@ -613,7 +613,7 @@ func TestDebugURLFormatting(t *testing.T) { ParseTabletURLTemplateFromFlag() tablet := topo.NewTablet(0, "cell", "host.dc.domain") - ts := []*TabletStats{ + ts := []*LegacyTabletStats{ { Key: "a", Tablet: tablet, @@ -624,7 +624,7 @@ func TestDebugURLFormatting(t *testing.T) { TabletExternallyReparentedTimestamp: 0, }, } - tcs := &TabletsCacheStatus{ + tcs := &LegacyTabletsCacheStatus{ Cell: "cell", Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, TabletsStats: ts, @@ -635,7 +635,7 @@ func TestDebugURLFormatting(t *testing.T) { t.Fatalf("error parsing template: %v", err) } wr := &bytes.Buffer{} - if err := templ.Execute(wr, []*TabletsCacheStatus{tcs}); err != nil { + if err := templ.Execute(wr, []*LegacyTabletsCacheStatus{tcs}); err != nil { t.Fatalf("error executing template: %v", err) } expectedURL := `"https://host.bastion.cell.corp"` @@ -645,14 +645,14 @@ func TestDebugURLFormatting(t *testing.T) { } type listener struct { - output chan *TabletStats + output chan *LegacyTabletStats } func newListener() *listener { - return &listener{output: make(chan *TabletStats, 2)} + return &listener{output: make(chan *LegacyTabletStats, 2)} } -func (l *listener) StatsUpdate(ts *TabletStats) { +func (l *listener) StatsUpdate(ts *LegacyTabletStats) { l.output <- ts } diff --git a/go/vt/discovery/tablet_stats_cache.go b/go/vt/discovery/legacy_tablet_stats_cache.go similarity index 73% rename from go/vt/discovery/tablet_stats_cache.go rename to go/vt/discovery/legacy_tablet_stats_cache.go index 14771e7aec9..dc8d981cedb 100644 --- a/go/vt/discovery/tablet_stats_cache.go +++ b/go/vt/discovery/legacy_tablet_stats_cache.go @@ -28,8 +28,8 @@ import ( "vitess.io/vitess/go/vt/topo/topoproto" ) -// TabletStatsCache is a HealthCheckStatsListener that keeps both the -// current list of available TabletStats, and a serving list: +// LegacyTabletStatsCache is a LegacyHealthCheckStatsListener that keeps both the +// current list of available LegacyTabletStats, and a serving list: // - for master tablets, only the current master is kept. // - for non-master tablets, we filter the list using FilterByReplicationLag. // It keeps entries for all tablets in the cell(s) it's configured to serve for, @@ -39,7 +39,7 @@ import ( // Also note the cache may not have the last entry received by the tablet. // For instance, if a tablet was healthy, and is still healthy, we do not // keep its new update. -type TabletStatsCache struct { +type LegacyTabletStatsCache struct { // cell is the cell we are keeping all tablets for. // Note we keep track of all master tablets in all cells. cell string @@ -49,26 +49,26 @@ type TabletStatsCache struct { // entries in the entries map. mu sync.RWMutex // entries maps from keyspace/shard/tabletType to our cache. - entries map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry + entries map[string]map[string]map[topodatapb.TabletType]*legacyTabletStatsCacheEntry // tsm is a helper to broadcast aggregate stats. tsm srvtopo.TargetStatsMultiplexer // cellAliases is a cache of cell aliases cellAliases map[string]string } -// tabletStatsCacheEntry is the per keyspace/shard/tabletType -// entry of the in-memory map for TabletStatsCache. -type tabletStatsCacheEntry struct { +// legacyTabletStatsCacheEntry is the per keyspace/shard/tabletType +// entry of the in-memory map for LegacyTabletStatsCache. +type legacyTabletStatsCacheEntry struct { // mu protects the rest of this structure. mu sync.RWMutex // all has the valid tablets, indexed by TabletToMapKey(ts.Tablet), - // as it is the index used by HealthCheck. - all map[string]*TabletStats + // as it is the index used by LegacyHealthCheck. + all map[string]*LegacyTabletStats // healthy only has the healthy ones. - healthy []*TabletStats + healthy []*LegacyTabletStats } -func (e *tabletStatsCacheEntry) updateHealthyMapForMaster(ts *TabletStats) { +func (e *legacyTabletStatsCacheEntry) updateHealthyMapForMaster(ts *LegacyTabletStats) { if ts.Up { // We have an Up master. if len(e.healthy) == 0 { @@ -103,31 +103,31 @@ func (e *tabletStatsCacheEntry) updateHealthyMapForMaster(ts *TabletStats) { } } -// NewTabletStatsCache creates a TabletStatsCache, and registers -// it as HealthCheckStatsListener of the provided healthcheck. +// NewLegacyTabletStatsCache creates a LegacyTabletStatsCache, and registers +// it as LegacyHealthCheckStatsListener of the provided healthcheck. // Note we do the registration in this code to guarantee we call // SetListener with sendDownEvents=true, as we need these events // to maintain the integrity of our cache. -func NewTabletStatsCache(hc HealthCheck, ts *topo.Server, cell string) *TabletStatsCache { - return newTabletStatsCache(hc, ts, cell, true /* setListener */) +func NewLegacyTabletStatsCache(hc LegacyHealthCheck, ts *topo.Server, cell string) *LegacyTabletStatsCache { + return newLegacyTabletStatsCache(hc, ts, cell, true /* setListener */) } -// NewTabletStatsCacheDoNotSetListener is identical to NewTabletStatsCache +// NewTabletStatsCacheDoNotSetListener is identical to NewLegacyTabletStatsCache // but does not automatically set the returned object as listener for "hc". -// Instead, it's up to the caller to ensure that TabletStatsCache.StatsUpdate() +// Instead, it's up to the caller to ensure that LegacyTabletStatsCache.StatsUpdate() // gets called properly. This is useful for chaining multiple listeners. // When the caller sets its own listener on "hc", they must make sure that they // set the parameter "sendDownEvents" to "true" or this cache won't properly // remove tablets whose tablet type changes. -func NewTabletStatsCacheDoNotSetListener(ts *topo.Server, cell string) *TabletStatsCache { - return newTabletStatsCache(nil, ts, cell, false /* setListener */) +func NewTabletStatsCacheDoNotSetListener(ts *topo.Server, cell string) *LegacyTabletStatsCache { + return newLegacyTabletStatsCache(nil, ts, cell, false /* setListener */) } -func newTabletStatsCache(hc HealthCheck, ts *topo.Server, cell string, setListener bool) *TabletStatsCache { - tc := &TabletStatsCache{ +func newLegacyTabletStatsCache(hc LegacyHealthCheck, ts *topo.Server, cell string, setListener bool) *LegacyTabletStatsCache { + tc := &LegacyTabletStatsCache{ cell: cell, ts: ts, - entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), + entries: make(map[string]map[string]map[topodatapb.TabletType]*legacyTabletStatsCacheEntry), tsm: srvtopo.NewTargetStatsMultiplexer(), cellAliases: make(map[string]string), } @@ -140,9 +140,9 @@ func newTabletStatsCache(hc HealthCheck, ts *topo.Server, cell string, setListen return tc } -// getEntry returns an existing tabletStatsCacheEntry in the cache, or nil +// getEntry returns an existing legacyTabletStatsCacheEntry in the cache, or nil // if the entry does not exist. It only takes a Read lock on mu. -func (tc *TabletStatsCache) getEntry(keyspace, shard string, tabletType topodatapb.TabletType) *tabletStatsCacheEntry { +func (tc *LegacyTabletStatsCache) getEntry(keyspace, shard string, tabletType topodatapb.TabletType) *legacyTabletStatsCacheEntry { tc.mu.RLock() defer tc.mu.RUnlock() @@ -156,9 +156,9 @@ func (tc *TabletStatsCache) getEntry(keyspace, shard string, tabletType topodata return nil } -// getOrCreateEntry returns an existing tabletStatsCacheEntry from the cache, +// getOrCreateEntry returns an existing legacyTabletStatsCacheEntry from the cache, // or creates it if it doesn't exist. -func (tc *TabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStatsCacheEntry { +func (tc *LegacyTabletStatsCache) getOrCreateEntry(target *querypb.Target) *legacyTabletStatsCacheEntry { // Fast path (most common path too): Read-lock, return the entry. if e := tc.getEntry(target.Keyspace, target.Shard, target.TabletType); e != nil { return e @@ -170,25 +170,25 @@ func (tc *TabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStat s, ok := tc.entries[target.Keyspace] if !ok { - s = make(map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry) + s = make(map[string]map[topodatapb.TabletType]*legacyTabletStatsCacheEntry) tc.entries[target.Keyspace] = s } t, ok := s[target.Shard] if !ok { - t = make(map[topodatapb.TabletType]*tabletStatsCacheEntry) + t = make(map[topodatapb.TabletType]*legacyTabletStatsCacheEntry) s[target.Shard] = t } e, ok := t[target.TabletType] if !ok { - e = &tabletStatsCacheEntry{ - all: make(map[string]*TabletStats), + e = &legacyTabletStatsCacheEntry{ + all: make(map[string]*LegacyTabletStats), } t[target.TabletType] = e } return e } -func (tc *TabletStatsCache) getAliasByCell(cell string) string { +func (tc *LegacyTabletStatsCache) getAliasByCell(cell string) string { tc.mu.Lock() defer tc.mu.Unlock() @@ -202,8 +202,8 @@ func (tc *TabletStatsCache) getAliasByCell(cell string) string { return alias } -// StatsUpdate is part of the HealthCheckStatsListener interface. -func (tc *TabletStatsCache) StatsUpdate(ts *TabletStats) { +// StatsUpdate is part of the LegacyHealthCheckStatsListener interface. +func (tc *LegacyTabletStatsCache) StatsUpdate(ts *LegacyTabletStats) { if ts.Target.TabletType != topodatapb.TabletType_MASTER && ts.Tablet.Alias.Cell != tc.cell && tc.getAliasByCell(ts.Tablet.Alias.Cell) != tc.getAliasByCell(tc.cell) { @@ -245,7 +245,7 @@ func (tc *TabletStatsCache) StatsUpdate(ts *TabletStats) { } // Update our healthy list. - var allArray []*TabletStats + var allArray []*LegacyTabletStats if ts.Target.TabletType == topodatapb.TabletType_MASTER { // The healthy list is different for TabletType_MASTER: we // only keep the most recent one. @@ -259,7 +259,7 @@ func (tc *TabletStatsCache) StatsUpdate(ts *TabletStats) { } // Now we need to do some work. Recompute our healthy list. - allArray = make([]*TabletStats, 0, len(e.all)) + allArray = make([]*LegacyTabletStats, 0, len(e.all)) for _, s := range e.all { allArray = append(allArray, s) } @@ -269,7 +269,7 @@ func (tc *TabletStatsCache) StatsUpdate(ts *TabletStats) { // GetTabletStats returns the full list of available targets. // The returned array is owned by the caller. -func (tc *TabletStatsCache) GetTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats { +func (tc *LegacyTabletStatsCache) GetTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []LegacyTabletStats { e := tc.getEntry(keyspace, shard, tabletType) if e == nil { return nil @@ -277,7 +277,7 @@ func (tc *TabletStatsCache) GetTabletStats(keyspace, shard string, tabletType to e.mu.RLock() defer e.mu.RUnlock() - result := make([]TabletStats, 0, len(e.all)) + result := make([]LegacyTabletStats, 0, len(e.all)) for _, s := range e.all { result = append(result, *s) } @@ -288,7 +288,7 @@ func (tc *TabletStatsCache) GetTabletStats(keyspace, shard string, tabletType to // The returned array is owned by the caller. // For TabletType_MASTER, this will only return at most one entry, // the most recent tablet of type master. -func (tc *TabletStatsCache) GetHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats { +func (tc *LegacyTabletStatsCache) GetHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []LegacyTabletStats { e := tc.getEntry(keyspace, shard, tabletType) if e == nil { return nil @@ -296,7 +296,7 @@ func (tc *TabletStatsCache) GetHealthyTabletStats(keyspace, shard string, tablet e.mu.RLock() defer e.mu.RUnlock() - result := make([]TabletStats, len(e.healthy)) + result := make([]LegacyTabletStats, len(e.healthy)) for i, ts := range e.healthy { result[i] = *ts } @@ -304,12 +304,12 @@ func (tc *TabletStatsCache) GetHealthyTabletStats(keyspace, shard string, tablet } // ResetForTesting is for use in tests only. -func (tc *TabletStatsCache) ResetForTesting() { +func (tc *LegacyTabletStatsCache) ResetForTesting() { tc.mu.Lock() defer tc.mu.Unlock() - tc.entries = make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry) + tc.entries = make(map[string]map[string]map[topodatapb.TabletType]*legacyTabletStatsCacheEntry) } // Compile-time interface check. -var _ HealthCheckStatsListener = (*TabletStatsCache)(nil) +var _ LegacyHealthCheckStatsListener = (*LegacyTabletStatsCache)(nil) diff --git a/go/vt/discovery/tablet_stats_cache_test.go b/go/vt/discovery/legacy_tablet_stats_cache_test.go similarity index 94% rename from go/vt/discovery/tablet_stats_cache_test.go rename to go/vt/discovery/legacy_tablet_stats_cache_test.go index 9897d503db7..97bf9eff4dd 100644 --- a/go/vt/discovery/tablet_stats_cache_test.go +++ b/go/vt/discovery/legacy_tablet_stats_cache_test.go @@ -27,7 +27,7 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -// TestTabletStatsCache tests the functionality of the TabletStatsCache class. +// TestTabletStatsCache tests the functionality of the LegacyTabletStatsCache class. func TestTabletStatsCache(t *testing.T) { ts := memorytopo.NewServer("cell", "cell1", "cell2") @@ -47,13 +47,13 @@ func TestTabletStatsCache(t *testing.T) { defer ts.DeleteCellsAlias(context.Background(), "region2") - // We want to unit test TabletStatsCache without a full-blown - // HealthCheck object, so we can't call NewTabletStatsCache. + // We want to unit test LegacyTabletStatsCache without a full-blown + // LegacyHealthCheck object, so we can't call NewLegacyTabletStatsCache. // So we just construct this object here. - tsc := &TabletStatsCache{ + tsc := &LegacyTabletStatsCache{ cell: "cell", ts: ts, - entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), + entries: make(map[string]map[string]map[topodatapb.TabletType]*legacyTabletStatsCacheEntry), cellAliases: make(map[string]string), } @@ -65,7 +65,7 @@ func TestTabletStatsCache(t *testing.T) { // add a tablet tablet1 := topo.NewTablet(10, "cell", "host1") - ts1 := &TabletStats{ + ts1 := &LegacyTabletStats{ Key: "t1", Tablet: tablet1, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -86,7 +86,7 @@ func TestTabletStatsCache(t *testing.T) { } // update stats with a change that won't change health array - stillHealthyTs1 := &TabletStats{ + stillHealthyTs1 := &LegacyTabletStats{ Key: "t1", Tablet: tablet1, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -107,7 +107,7 @@ func TestTabletStatsCache(t *testing.T) { } // update stats with a change that will change arrays - notHealthyTs1 := &TabletStats{ + notHealthyTs1 := &LegacyTabletStats{ Key: "t1", Tablet: tablet1, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -129,7 +129,7 @@ func TestTabletStatsCache(t *testing.T) { // add a second tablet tablet2 := topo.NewTablet(11, "cell", "host2") - ts2 := &TabletStats{ + ts2 := &LegacyTabletStats{ Key: "t2", Tablet: tablet2, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -232,7 +232,7 @@ func TestTabletStatsCache(t *testing.T) { // add a third tablet as slave in diff cell, same region tablet3 := topo.NewTablet(12, "cell1", "host3") - ts3 := &TabletStats{ + ts3 := &LegacyTabletStats{ Key: "t3", Tablet: tablet3, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -253,7 +253,7 @@ func TestTabletStatsCache(t *testing.T) { // add a 4th slave tablet in a diff cell, diff region tablet4 := topo.NewTablet(13, "cell2", "host4") - ts4 := &TabletStats{ + ts4 := &LegacyTabletStats{ Key: "t4", Tablet: tablet4, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, diff --git a/go/vt/discovery/tablet_stats_cache_wait.go b/go/vt/discovery/legacy_tablet_stats_cache_wait.go similarity index 82% rename from go/vt/discovery/tablet_stats_cache_wait.go rename to go/vt/discovery/legacy_tablet_stats_cache_wait.go index 1b1123a7439..d984ff7d1ce 100644 --- a/go/vt/discovery/tablet_stats_cache_wait.go +++ b/go/vt/discovery/legacy_tablet_stats_cache_wait.go @@ -33,7 +33,7 @@ var ( // WaitForTablets waits for at least one tablet in the given // keyspace / shard / tablet type before returning. The tablets do not // have to be healthy. It will return ctx.Err() if the context is canceled. -func (tc *TabletStatsCache) WaitForTablets(ctx context.Context, keyspace, shard string, tabletType topodatapb.TabletType) error { +func (tc *LegacyTabletStatsCache) WaitForTablets(ctx context.Context, keyspace, shard string, tabletType topodatapb.TabletType) error { targets := []*querypb.Target{ { Keyspace: keyspace, @@ -48,12 +48,12 @@ func (tc *TabletStatsCache) WaitForTablets(ctx context.Context, keyspace, shard // each given target before returning. // It will return ctx.Err() if the context is canceled. // It will return an error if it can't read the necessary topology records. -func (tc *TabletStatsCache) WaitForAllServingTablets(ctx context.Context, targets []*querypb.Target) error { +func (tc *LegacyTabletStatsCache) WaitForAllServingTablets(ctx context.Context, targets []*querypb.Target) error { return tc.waitForTablets(ctx, targets, true) } // waitForTablets is the internal method that polls for tablets. -func (tc *TabletStatsCache) waitForTablets(ctx context.Context, targets []*querypb.Target, requireServing bool) error { +func (tc *LegacyTabletStatsCache) waitForTablets(ctx context.Context, targets []*querypb.Target, requireServing bool) error { for { // We nil targets as we find them. allPresent := true @@ -62,7 +62,7 @@ func (tc *TabletStatsCache) waitForTablets(ctx context.Context, targets []*query continue } - var stats []TabletStats + var stats []LegacyTabletStats if requireServing { stats = tc.GetHealthyTabletStats(target.Keyspace, target.Shard, target.TabletType) } else { @@ -92,7 +92,7 @@ func (tc *TabletStatsCache) waitForTablets(ctx context.Context, targets []*query } // WaitByFilter waits for at least one tablet based on the filter function. -func (tc *TabletStatsCache) WaitByFilter(ctx context.Context, keyspace, shard string, tabletTypes []topodatapb.TabletType, filter func([]TabletStats) []TabletStats) error { +func (tc *LegacyTabletStatsCache) WaitByFilter(ctx context.Context, keyspace, shard string, tabletTypes []topodatapb.TabletType, filter func([]LegacyTabletStats) []LegacyTabletStats) error { for { for _, tt := range tabletTypes { stats := tc.GetTabletStats(keyspace, shard, tt) diff --git a/go/vt/discovery/tablet_stats_cache_wait_test.go b/go/vt/discovery/legacy_tablet_stats_cache_wait_test.go similarity index 95% rename from go/vt/discovery/tablet_stats_cache_wait_test.go rename to go/vt/discovery/legacy_tablet_stats_cache_wait_test.go index 0cf309931c1..1eb7c4df470 100644 --- a/go/vt/discovery/tablet_stats_cache_wait_test.go +++ b/go/vt/discovery/legacy_tablet_stats_cache_wait_test.go @@ -38,8 +38,8 @@ func TestWaitForTablets(t *testing.T) { input := make(chan *querypb.StreamHealthResponse) createFakeConn(tablet, input) - hc := NewHealthCheck(1*time.Millisecond, 1*time.Hour) - tsc := NewTabletStatsCache(hc, nil, "cell") + hc := NewLegacyHealthCheck(1*time.Millisecond, 1*time.Hour) + tsc := NewLegacyTabletStatsCache(hc, nil, "cell") hc.AddTablet(tablet, "") // this should time out diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index dea0f5f5152..c2bbe9569a1 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -31,20 +31,20 @@ var ( legacyReplicationLagAlgorithm = flag.Bool("legacy_replication_lag_algorithm", true, "use the legacy algorithm when selecting the vttablets for serving") ) -// IsReplicationLagHigh verifies that the given TabletStats refers to a tablet with high +// IsReplicationLagHigh verifies that the given LegacyTabletStats refers to a tablet with high // replication lag, i.e. higher than the configured discovery_low_replication_lag flag. -func IsReplicationLagHigh(tabletStats *TabletStats) bool { +func IsReplicationLagHigh(tabletStats *LegacyTabletStats) bool { return float64(tabletStats.Stats.SecondsBehindMaster) > lowReplicationLag.Seconds() } -// IsReplicationLagVeryHigh verifies that the given TabletStats refers to a tablet with very high +// IsReplicationLagVeryHigh verifies that the given LegacyTabletStats refers to a tablet with very high // replication lag, i.e. higher than the configured discovery_high_replication_lag_minimum_serving flag. -func IsReplicationLagVeryHigh(tabletStats *TabletStats) bool { +func IsReplicationLagVeryHigh(tabletStats *LegacyTabletStats) bool { return float64(tabletStats.Stats.SecondsBehindMaster) > highReplicationLagMinServing.Seconds() } -// FilterByReplicationLag filters the list of TabletStats by TabletStats.Stats.SecondsBehindMaster. -// Note that TabletStats that is non-serving or has error is ignored. +// FilterByReplicationLag filters the list of LegacyTabletStats by LegacyTabletStats.Stats.SecondsBehindMaster. +// Note that LegacyTabletStats that is non-serving or has error is ignored. // // The simplified logic: // - Return tablets that have lag <= lowReplicationLag. @@ -68,7 +68,7 @@ func IsReplicationLagVeryHigh(tabletStats *TabletStats) bool { // The default for this is 2h, same as the discovery_high_replication_lag_minimum_serving here. // * degraded_threshold: this is only used by vttablet for display. It should match // discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. -func FilterByReplicationLag(tabletStatsList []*TabletStats) []*TabletStats { +func FilterByReplicationLag(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { if !*legacyReplicationLagAlgorithm { return filterByLag(tabletStatsList) } @@ -82,7 +82,7 @@ func FilterByReplicationLag(tabletStatsList []*TabletStats) []*TabletStats { return res } -func filterByLag(tabletStatsList []*TabletStats) []*TabletStats { +func filterByLag(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { list := make([]tabletLagSnapshot, 0, len(tabletStatsList)) // filter non-serving tablets and those with very high replication lag for _, ts := range tabletStatsList { @@ -99,7 +99,7 @@ func filterByLag(tabletStatsList []*TabletStats) []*TabletStats { sort.Sort(byReplag(list)) // Pick those with low replication lag, but at least minNumTablets tablets regardless. - res := make([]*TabletStats, 0, len(list)) + res := make([]*LegacyTabletStats, 0, len(list)) for i := 0; i < len(list); i++ { if !IsReplicationLagHigh(list[i].ts) || i < *minNumTablets { res = append(res, list[i].ts) @@ -108,8 +108,8 @@ func filterByLag(tabletStatsList []*TabletStats) []*TabletStats { return res } -func filterByLagWithLegacyAlgorithm(tabletStatsList []*TabletStats) []*TabletStats { - list := make([]*TabletStats, 0, len(tabletStatsList)) +func filterByLagWithLegacyAlgorithm(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { + list := make([]*LegacyTabletStats, 0, len(tabletStatsList)) // filter non-serving tablets for _, ts := range tabletStatsList { if !ts.Serving || ts.LastError != nil || ts.Stats == nil { @@ -133,7 +133,7 @@ func filterByLagWithLegacyAlgorithm(tabletStatsList []*TabletStats) []*TabletSta } // filter those affecting "mean" lag significantly // calculate mean for all tablets - res := make([]*TabletStats, 0, len(list)) + res := make([]*LegacyTabletStats, 0, len(list)) m, _ := mean(list, -1) for i, ts := range list { // calculate mean by excluding ith tablet @@ -175,7 +175,7 @@ func filterByLagWithLegacyAlgorithm(tabletStatsList []*TabletStats) []*TabletSta sort.Sort(byReplag(snapshots)) // Pick the first minNumTablets tablets. - res = make([]*TabletStats, 0, *minNumTablets) + res = make([]*LegacyTabletStats, 0, *minNumTablets) for i := 0; i < min(*minNumTablets, len(snapshots)); i++ { res = append(res, snapshots[i].ts) } @@ -190,7 +190,7 @@ func min(a, b int) int { } type tabletLagSnapshot struct { - ts *TabletStats + ts *LegacyTabletStats replag uint32 } type byReplag []tabletLagSnapshot @@ -201,7 +201,7 @@ func (a byReplag) Less(i, j int) bool { return a[i].replag < a[j].replag } // mean calculates the mean value over the given list, // while excluding the item with the specified index. -func mean(tabletStatsList []*TabletStats, idxExclude int) (uint64, error) { +func mean(tabletStatsList []*LegacyTabletStats, idxExclude int) (uint64, error) { var sum uint64 var count uint64 for i, ts := range tabletStatsList { @@ -217,9 +217,9 @@ func mean(tabletStatsList []*TabletStats, idxExclude int) (uint64, error) { return sum / count, nil } -// TrivialStatsUpdate returns true iff the old and new TabletStats +// TrivialStatsUpdate returns true iff the old and new LegacyTabletStats // haven't changed enough to warrant re-calling FilterByReplicationLag. -func TrivialStatsUpdate(o, n *TabletStats) bool { +func TrivialStatsUpdate(o, n *LegacyTabletStats) bool { // Skip replag filter when replag remains in the low rep lag range, // which should be the case majority of the time. lowRepLag := lowReplicationLag.Seconds() diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index 40e184fc8b6..f7958b7d3af 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -36,17 +36,17 @@ func testSetLegacyReplicationLagAlgorithm(newLegacy bool) { func TestFilterByReplicationLagUnhealthy(t *testing.T) { // 1 healthy serving tablet, 1 not healhty - ts1 := &TabletStats{ + ts1 := &LegacyTabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{}, } - ts2 := &TabletStats{ + ts2 := &LegacyTabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: false, Stats: &querypb.RealtimeStats{}, } - got := FilterByReplicationLag([]*TabletStats{ts1, ts2}) + got := FilterByReplicationLag([]*LegacyTabletStats{ts1, ts2}) if len(got) != 1 { t.Errorf("len(FilterByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}])) = %v, want 1", len(got)) } @@ -107,9 +107,9 @@ func TestFilterByReplicationLag(t *testing.T) { } for _, tc := range cases { - lts := make([]*TabletStats, len(tc.input)) + lts := make([]*LegacyTabletStats, len(tc.input)) for i, lag := range tc.input { - lts[i] = &TabletStats{ + lts[i] = &LegacyTabletStats{ Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, @@ -216,9 +216,9 @@ func TestFilterByReplicationLagWithLegacyAlgorithm(t *testing.T) { } for _, tc := range cases { - lts := make([]*TabletStats, len(tc.input)) + lts := make([]*LegacyTabletStats, len(tc.input)) for i, lag := range tc.input { - lts[i] = &TabletStats{ + lts[i] = &LegacyTabletStats{ Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, @@ -241,52 +241,52 @@ func TestFilterByReplicationLagThreeTabletMin(t *testing.T) { // Use at least 3 tablets if possible testSetMinNumTablets(3) // lags of (1s, 1s, 10m, 11m) - returns at least32 items where the slightly delayed ones that are returned are the 10m and 11m ones. - ts1 := &TabletStats{ + ts1 := &LegacyTabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &TabletStats{ + ts2 := &LegacyTabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts3 := &TabletStats{ + ts3 := &LegacyTabletStats{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts4 := &TabletStats{ + ts4 := &LegacyTabletStats{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - got := FilterByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4}) + got := FilterByReplicationLag([]*LegacyTabletStats{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts1) || !got[1].DeepEqual(ts2) || !got[2].DeepEqual(ts3) { t.Errorf("FilterByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } // lags of (11m, 10m, 1s, 1s) - reordered tablets returns the same 3 items where the slightly delayed one that is returned is the 10m and 11m ones. - ts1 = &TabletStats{ + ts1 = &LegacyTabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - ts2 = &TabletStats{ + ts2 = &LegacyTabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts3 = &TabletStats{ + ts3 = &LegacyTabletStats{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts4 = &TabletStats{ + ts4 = &LegacyTabletStats{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - got = FilterByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4}) + got = FilterByReplicationLag([]*LegacyTabletStats{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts3) || !got[1].DeepEqual(ts4) || !got[2].DeepEqual(ts2) { t.Errorf("FilterByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } @@ -298,32 +298,32 @@ func TestFilterByReplicationLagOneTabletMin(t *testing.T) { // Use at least 1 tablets if possible testSetMinNumTablets(1) // lags of (1s, 100m) - return only healthy tablet if that is all that is available. - ts1 := &TabletStats{ + ts1 := &LegacyTabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &TabletStats{ + ts2 := &LegacyTabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got := FilterByReplicationLag([]*TabletStats{ts1, ts2}) + got := FilterByReplicationLag([]*LegacyTabletStats{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { t.Errorf("FilterByReplicationLag([1s, 100m]) = %+v, want [1s]", got) } // lags of (1m, 100m) - return only healthy tablet if that is all that is healthy enough. - ts1 = &TabletStats{ + ts1 = &LegacyTabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60}, } - ts2 = &TabletStats{ + ts2 = &LegacyTabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got = FilterByReplicationLag([]*TabletStats{ts1, ts2}) + got = FilterByReplicationLag([]*LegacyTabletStats{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { t.Errorf("FilterByReplicationLag([1m, 100m]) = %+v, want [1m]", got) } @@ -357,12 +357,12 @@ func TestTrivialStatsUpdate(t *testing.T) { } for _, c := range cases { - o := &TabletStats{ + o := &LegacyTabletStats{ Stats: &querypb.RealtimeStats{ SecondsBehindMaster: c.o, }, } - n := &TabletStats{ + n := &LegacyTabletStats{ Stats: &querypb.RealtimeStats{ SecondsBehindMaster: c.n, }, diff --git a/go/vt/discovery/tablet_picker.go b/go/vt/discovery/tablet_picker.go index 1cd75ab60c2..05e998e1111 100644 --- a/go/vt/discovery/tablet_picker.go +++ b/go/vt/discovery/tablet_picker.go @@ -36,9 +36,9 @@ type TabletPicker struct { shard string tabletTypes []topodatapb.TabletType - healthCheck HealthCheck + healthCheck LegacyHealthCheck watcher *TopologyWatcher - statsCache *TabletStatsCache + statsCache *LegacyTabletStatsCache } // NewTabletPicker returns a TabletPicker. @@ -49,8 +49,8 @@ func NewTabletPicker(ctx context.Context, ts *topo.Server, cell, keyspace, shard } // These have to be initialized in the following sequence (watcher must be last). - healthCheck := NewHealthCheck(healthcheckRetryDelay, healthcheckTimeout) - statsCache := NewTabletStatsCache(healthCheck, ts, cell) + healthCheck := NewLegacyHealthCheck(healthcheckRetryDelay, healthcheckTimeout) + statsCache := NewLegacyTabletStatsCache(healthCheck, ts, cell) watcher := NewShardReplicationWatcher(ctx, ts, healthCheck, cell, keyspace, shard, healthcheckTopologyRefresh, DefaultTopoReadConcurrency) return &TabletPicker{ @@ -73,7 +73,7 @@ func (tp *TabletPicker) PickForStreaming(ctx context.Context) (*topodatapb.Table } // Refilter the tablets list based on the same criteria. - var addrs []TabletStats + var addrs []LegacyTabletStats for _, tabletType := range tp.tabletTypes { list := RemoveUnhealthyTablets(tp.statsCache.GetTabletStats(tp.keyspace, tp.shard, tabletType)) addrs = append(addrs, list...) diff --git a/go/vt/discovery/topology_watcher.go b/go/vt/discovery/topology_watcher.go index b623457663c..f0e55ba9f0d 100644 --- a/go/vt/discovery/topology_watcher.go +++ b/go/vt/discovery/topology_watcher.go @@ -52,7 +52,7 @@ var ( "Operation", topologyWatcherOpListTablets, topologyWatcherOpGetTablet) ) -// TabletRecorder is the part of the HealthCheck interface that can +// TabletRecorder is the part of the LegacyHealthCheck interface that can // add or remove tablets. We define it as a sub-interface here so we // can add filters on tablets if needed. type TabletRecorder interface { diff --git a/go/vt/discovery/utils.go b/go/vt/discovery/utils.go index ac57b032ded..efebd898348 100644 --- a/go/vt/discovery/utils.go +++ b/go/vt/discovery/utils.go @@ -16,14 +16,22 @@ limitations under the License. package discovery +import ( + "sort" + "strings" + + "vitess.io/vitess/go/netutil" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + // This file contains helper filter methods to process the unfiltered list of -// tablets returned by HealthCheck.GetTabletStatsFrom*. +// tablets returned by LegacyHealthCheck.GetTabletStatsFrom*. // See also replicationlag.go for a more sophisicated filter used by vtgate. // RemoveUnhealthyTablets filters all unhealthy tablets out. // NOTE: Non-serving tablets are considered healthy. -func RemoveUnhealthyTablets(tabletStatsList []TabletStats) []TabletStats { - result := make([]TabletStats, 0, len(tabletStatsList)) +func RemoveUnhealthyTablets(tabletStatsList []LegacyTabletStats) []LegacyTabletStats { + result := make([]LegacyTabletStats, 0, len(tabletStatsList)) for _, ts := range tabletStatsList { // Note we do not check the 'Serving' flag here. // This is mainly to avoid the case where we run a vtworker Diff between a @@ -37,3 +45,15 @@ func RemoveUnhealthyTablets(tabletStatsList []TabletStats) []TabletStats { } return result } + +// TabletToMapKey creates a key to the map from tablet's host and ports. +// It should only be used in discovery and related module. +func TabletToMapKey(tablet *topodatapb.Tablet) string { + parts := make([]string, 0, 1) + for name, port := range tablet.PortMap { + parts = append(parts, netutil.JoinHostPort(name, port)) + } + sort.Strings(parts) + parts = append([]string{tablet.Hostname}, parts...) + return strings.Join(parts, ",") +} diff --git a/go/vt/discovery/utils_test.go b/go/vt/discovery/utils_test.go index 4a0266072c9..c9db72e28be 100644 --- a/go/vt/discovery/utils_test.go +++ b/go/vt/discovery/utils_test.go @@ -27,36 +27,36 @@ import ( func TestRemoveUnhealthyTablets(t *testing.T) { var testcases = []struct { desc string - input []TabletStats - want []TabletStats + input []LegacyTabletStats + want []LegacyTabletStats }{{ desc: "tablets missing Stats", - input: []TabletStats{replica(1), replica(2)}, - want: []TabletStats{}, + input: []LegacyTabletStats{replica(1), replica(2)}, + want: []LegacyTabletStats{}, }, { desc: "all tablets healthy", - input: []TabletStats{healthy(replica(1)), healthy(replica(2))}, - want: []TabletStats{healthy(replica(1)), healthy(replica(2))}, + input: []LegacyTabletStats{healthy(replica(1)), healthy(replica(2))}, + want: []LegacyTabletStats{healthy(replica(1)), healthy(replica(2))}, }, { desc: "one unhealthy tablet (error)", - input: []TabletStats{healthy(replica(1)), unhealthyError(replica(2))}, - want: []TabletStats{healthy(replica(1))}, + input: []LegacyTabletStats{healthy(replica(1)), unhealthyError(replica(2))}, + want: []LegacyTabletStats{healthy(replica(1))}, }, { desc: "one error tablet", - input: []TabletStats{healthy(replica(1)), unhealthyLastError(replica(2))}, - want: []TabletStats{healthy(replica(1))}, + input: []LegacyTabletStats{healthy(replica(1)), unhealthyLastError(replica(2))}, + want: []LegacyTabletStats{healthy(replica(1))}, }, { desc: "one unhealthy tablet (lag)", - input: []TabletStats{healthy(replica(1)), unhealthyLag(replica(2))}, - want: []TabletStats{healthy(replica(1))}, + input: []LegacyTabletStats{healthy(replica(1)), unhealthyLag(replica(2))}, + want: []LegacyTabletStats{healthy(replica(1))}, }, { desc: "no filtering by tablet type", - input: []TabletStats{healthy(master(1)), healthy(replica(2)), healthy(rdonly(3))}, - want: []TabletStats{healthy(master(1)), healthy(replica(2)), healthy(rdonly(3))}, + input: []LegacyTabletStats{healthy(master(1)), healthy(replica(2)), healthy(rdonly(3))}, + want: []LegacyTabletStats{healthy(master(1)), healthy(replica(2)), healthy(rdonly(3))}, }, { desc: "non-serving tablets won't be removed", - input: []TabletStats{notServing(healthy(replica(1)))}, - want: []TabletStats{notServing(healthy(replica(1)))}, + input: []LegacyTabletStats{notServing(healthy(replica(1)))}, + want: []LegacyTabletStats{notServing(healthy(replica(1)))}, }} for _, tc := range testcases { @@ -73,20 +73,20 @@ func TestRemoveUnhealthyTablets(t *testing.T) { } } -func master(uid uint32) TabletStats { +func master(uid uint32) LegacyTabletStats { return minimalTabletStats(uid, topodatapb.TabletType_MASTER) } -func replica(uid uint32) TabletStats { +func replica(uid uint32) LegacyTabletStats { return minimalTabletStats(uid, topodatapb.TabletType_REPLICA) } -func rdonly(uid uint32) TabletStats { +func rdonly(uid uint32) LegacyTabletStats { return minimalTabletStats(uid, topodatapb.TabletType_RDONLY) } -func minimalTabletStats(uid uint32, tabletType topodatapb.TabletType) TabletStats { - return TabletStats{ +func minimalTabletStats(uid uint32, tabletType topodatapb.TabletType) LegacyTabletStats { + return LegacyTabletStats{ Tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Uid: uid}, @@ -100,33 +100,33 @@ func minimalTabletStats(uid uint32, tabletType topodatapb.TabletType) TabletStat } } -func healthy(ts TabletStats) TabletStats { +func healthy(ts LegacyTabletStats) LegacyTabletStats { ts.Stats = &querypb.RealtimeStats{ SecondsBehindMaster: uint32(1), } return ts } -func unhealthyLag(ts TabletStats) TabletStats { +func unhealthyLag(ts LegacyTabletStats) LegacyTabletStats { ts.Stats = &querypb.RealtimeStats{ SecondsBehindMaster: uint32(3600), } return ts } -func unhealthyError(ts TabletStats) TabletStats { +func unhealthyError(ts LegacyTabletStats) LegacyTabletStats { ts.Stats = &querypb.RealtimeStats{ HealthError: "unhealthy", } return ts } -func unhealthyLastError(ts TabletStats) TabletStats { +func unhealthyLastError(ts LegacyTabletStats) LegacyTabletStats { ts.LastError = errors.New("err") return ts } -func notServing(ts TabletStats) TabletStats { +func notServing(ts LegacyTabletStats) LegacyTabletStats { ts.Serving = false return ts } diff --git a/go/vt/proto/query/query.pb.go b/go/vt/proto/query/query.pb.go index 45f9128cebd..260948bfaa4 100644 --- a/go/vt/proto/query/query.pb.go +++ b/go/vt/proto/query/query.pb.go @@ -3629,7 +3629,7 @@ type StreamHealthResponse struct { // realtime_stats contains information about the tablet status. // It is only filled in if the information is about a tablet. RealtimeStats *RealtimeStats `protobuf:"bytes,4,opt,name=realtime_stats,json=realtimeStats,proto3" json:"realtime_stats,omitempty"` - // tablet_alias is the alias of the sending tablet. The discovery/healthcheck.go + // tablet_alias is the alias of the sending tablet. The discovery/legacy_healthcheck.go // code uses it to verify that it's talking to the correct tablet and that it // hasn't changed in the meantime e.g. due to tablet restarts where ports or // ips have been reused but assigned differently. diff --git a/go/vt/schemamanager/schemaswap/schema_swap.go b/go/vt/schemamanager/schemaswap/schema_swap.go index 14e7161d919..2f7be38666b 100644 --- a/go/vt/schemamanager/schemaswap/schema_swap.go +++ b/go/vt/schemamanager/schemaswap/schema_swap.go @@ -136,7 +136,7 @@ type shardSchemaSwap struct { numTabletsSwapped int // tabletHealthCheck watches after the healthiness of all tablets in the shard. - tabletHealthCheck discovery.HealthCheck + tabletHealthCheck discovery.LegacyHealthCheck // tabletWatchers contains list of topology watchers monitoring changes in the shard // topology. There are several of them because the watchers are per-cell. tabletWatchers []*discovery.TopologyWatcher @@ -146,7 +146,7 @@ type shardSchemaSwap struct { allTabletsLock sync.RWMutex // allTablets is the list of all tablets on the shard mapped by the key provided // by discovery. The contents of the map is guarded by allTabletsLock. - allTablets map[string]*discovery.TabletStats + allTablets map[string]*discovery.LegacyTabletStats // healthWaitingTablet is a key (the same key as used in allTablets) of a tablet that // is currently being waited on to become healthy and to catch up with replication. // The variable is guarded by allTabletsLock. @@ -686,9 +686,9 @@ func (shardSwap *shardSchemaSwap) writeFinishedSwap() error { // all tablets on the shard. Function should be called before the start of the schema // swap process. func (shardSwap *shardSchemaSwap) startHealthWatchers(ctx context.Context) error { - shardSwap.allTablets = make(map[string]*discovery.TabletStats) + shardSwap.allTablets = make(map[string]*discovery.LegacyTabletStats) - shardSwap.tabletHealthCheck = discovery.NewHealthCheck(*vtctl.HealthcheckRetryDelay, *vtctl.HealthCheckTimeout) + shardSwap.tabletHealthCheck = discovery.NewLegacyHealthCheck(*vtctl.HealthcheckRetryDelay, *vtctl.HealthCheckTimeout) shardSwap.tabletHealthCheck.SetListener(shardSwap, true /* sendDownEvents */) topoServer := shardSwap.parent.topoServer @@ -748,9 +748,9 @@ func (shardSwap *shardSchemaSwap) stopHealthWatchers() { } } -// isTabletHealthy verifies that the given TabletStats represents a healthy tablet that is +// isTabletHealthy verifies that the given LegacyTabletStats represents a healthy tablet that is // caught up with replication to a serving level. -func isTabletHealthy(tabletStats *discovery.TabletStats) bool { +func isTabletHealthy(tabletStats *discovery.LegacyTabletStats) bool { return tabletStats.Stats.HealthError == "" && !discovery.IsReplicationLagHigh(tabletStats) } @@ -778,11 +778,11 @@ func (shardSwap *shardSchemaSwap) startWaitingOnUnhealthyTablet(tablet *topodata return shardSwap.healthWaitingChannel, nil } -// checkWaitingTabletHealthiness verifies whether the provided TabletStats represent the +// checkWaitingTabletHealthiness verifies whether the provided LegacyTabletStats represent the // tablet that is being waited to become healthy, and notifies the waiting go routine if // it is the tablet and if it is healthy now. // The function should be called with shardSwap.allTabletsLock mutex locked. -func (shardSwap *shardSchemaSwap) checkWaitingTabletHealthiness(tabletStats *discovery.TabletStats) { +func (shardSwap *shardSchemaSwap) checkWaitingTabletHealthiness(tabletStats *discovery.LegacyTabletStats) { if shardSwap.healthWaitingTablet == tabletStats.Key && isTabletHealthy(tabletStats) { close(*shardSwap.healthWaitingChannel) shardSwap.healthWaitingChannel = nil @@ -790,11 +790,11 @@ func (shardSwap *shardSchemaSwap) checkWaitingTabletHealthiness(tabletStats *dis } } -// StatsUpdate is the part of discovery.HealthCheckStatsListener interface. It makes sure +// StatsUpdate is the part of discovery.LegacyHealthCheckStatsListener interface. It makes sure // that when a change of tablet health happens it's recorded in allTablets list, and if // this is the tablet that is being waited for after restore, the function wakes up the // waiting go routine. -func (shardSwap *shardSchemaSwap) StatsUpdate(newTabletStats *discovery.TabletStats) { +func (shardSwap *shardSchemaSwap) StatsUpdate(newTabletStats *discovery.LegacyTabletStats) { shardSwap.allTabletsLock.Lock() defer shardSwap.allTabletsLock.Unlock() @@ -813,21 +813,21 @@ func (shardSwap *shardSchemaSwap) StatsUpdate(newTabletStats *discovery.TabletSt // getTabletList returns the list of all known tablets in the shard so that the caller // could operate with it without holding the allTabletsLock. -func (shardSwap *shardSchemaSwap) getTabletList() []discovery.TabletStats { +func (shardSwap *shardSchemaSwap) getTabletList() []discovery.LegacyTabletStats { shardSwap.allTabletsLock.RLock() defer shardSwap.allTabletsLock.RUnlock() - tabletList := make([]discovery.TabletStats, 0, len(shardSwap.allTablets)) + tabletList := make([]discovery.LegacyTabletStats, 0, len(shardSwap.allTablets)) for _, tabletStats := range shardSwap.allTablets { tabletList = append(tabletList, *tabletStats) } return tabletList } -// orderTabletsForSwap is an alias for the slice of TabletStats. It implements +// orderTabletsForSwap is an alias for the slice of LegacyTabletStats. It implements // sort.Interface interface so that it's possible to sort the array in the order // in which schema swap will propagate. -type orderTabletsForSwap []discovery.TabletStats +type orderTabletsForSwap []discovery.LegacyTabletStats // Len is part of sort.Interface interface. func (array orderTabletsForSwap) Len() int { @@ -839,11 +839,11 @@ func (array orderTabletsForSwap) Swap(i, j int) { array[i], array[j] = array[j], array[i] } -// getTabletTypeFromStats returns the tablet type saved in the TabletStats object. If there is Target -// data in the TabletStats object then the function returns TabletType from it because it will be more +// getTabletTypeFromStats returns the tablet type saved in the LegacyTabletStats object. If there is Target +// data in the LegacyTabletStats object then the function returns TabletType from it because it will be more // up-to-date. But if that's not available then it returns Tablet.Type which will contain data read // from the topology during initialization of health watchers. -func getTabletTypeFromStats(tabletStats *discovery.TabletStats) topodatapb.TabletType { +func getTabletTypeFromStats(tabletStats *discovery.LegacyTabletStats) topodatapb.TabletType { if tabletStats.Target == nil || tabletStats.Target.TabletType == topodatapb.TabletType_UNKNOWN { return tabletStats.Tablet.Type } @@ -857,7 +857,7 @@ func getTabletTypeFromStats(tabletStats *discovery.TabletStats) topodatapb.Table // then will go 'replica' tablets, and the first will be 'rdonly' and all other // non-replica and non-master types. The sorting order within each of those 5 buckets // doesn't matter. -func tabletSortIndex(tabletStats *discovery.TabletStats) int { +func tabletSortIndex(tabletStats *discovery.LegacyTabletStats) int { tabletType := getTabletTypeFromStats(tabletStats) switch { case tabletType == topodatapb.TabletType_MASTER: diff --git a/go/vt/throttler/demo/throttler_demo.go b/go/vt/throttler/demo/throttler_demo.go index 783a27b8afd..b19512a8d00 100644 --- a/go/vt/throttler/demo/throttler_demo.go +++ b/go/vt/throttler/demo/throttler_demo.go @@ -213,7 +213,7 @@ func (r *replica) stop() { type client struct { master *master - healthCheck discovery.HealthCheck + healthCheck discovery.LegacyHealthCheck throttler *throttler.Throttler stopChan chan struct{} @@ -226,7 +226,7 @@ func newClient(master *master, replica *replica) *client { log.Fatal(err) } - healthCheck := discovery.NewHealthCheck(5*time.Second, 1*time.Minute) + healthCheck := discovery.NewLegacyHealthCheck(5*time.Second, 1*time.Minute) c := &client{ master: master, healthCheck: healthCheck, @@ -273,10 +273,10 @@ func (c *client) stop() { c.throttler.Close() } -// StatsUpdate implements discovery.HealthCheckStatsListener. +// StatsUpdate implements discovery.LegacyHealthCheckStatsListener. // It gets called by the healthCheck instance every time a tablet broadcasts // a health update. -func (c *client) StatsUpdate(ts *discovery.TabletStats) { +func (c *client) StatsUpdate(ts *discovery.LegacyTabletStats) { // Ignore unless REPLICA or RDONLY. if ts.Target.TabletType != topodatapb.TabletType_REPLICA && ts.Target.TabletType != topodatapb.TabletType_RDONLY { return diff --git a/go/vt/throttler/max_replication_lag_module.go b/go/vt/throttler/max_replication_lag_module.go index e0219d40b76..857e8ba52ed 100644 --- a/go/vt/throttler/max_replication_lag_module.go +++ b/go/vt/throttler/max_replication_lag_module.go @@ -54,7 +54,7 @@ const ( // i.e. we'll ignore lag records with lower lag from other replicas while we're // waiting for the next record of this replica under test. type replicaUnderTest struct { - // key holds the discovery.TabletStats.Key value for the replica. + // key holds the discovery.LegacyTabletStats.Key value for the replica. key string alias string tabletType topodatapb.TabletType @@ -114,7 +114,7 @@ type MaxReplicationLagModule struct { // max rate calculation has changed. The field is immutable (set in Start().) rateUpdateChan chan<- struct{} - // lagRecords buffers the replication lag records received by the HealthCheck + // lagRecords buffers the replication lag records received by the LegacyHealthCheck // listener. ProcessRecords() will process them. lagRecords chan replicationLagRecord wg sync.WaitGroup @@ -240,7 +240,7 @@ func (m *MaxReplicationLagModule) resetConfiguration() { } // RecordReplicationLag records the current replication lag for processing. -func (m *MaxReplicationLagModule) RecordReplicationLag(t time.Time, ts *discovery.TabletStats) { +func (m *MaxReplicationLagModule) RecordReplicationLag(t time.Time, ts *discovery.LegacyTabletStats) { m.mutableConfigMu.Lock() if m.mutableConfig.MaxReplicationLagSec == ReplicationLagModuleDisabled { m.mutableConfigMu.Unlock() @@ -248,7 +248,7 @@ func (m *MaxReplicationLagModule) RecordReplicationLag(t time.Time, ts *discover } m.mutableConfigMu.Unlock() - // Buffer data point for now to unblock the HealthCheck listener and process + // Buffer data point for now to unblock the LegacyHealthCheck listener and process // it asynchronously in ProcessRecords(). m.lagRecords <- replicationLagRecord{t, *ts} } @@ -404,7 +404,7 @@ func (m *MaxReplicationLagModule) clearReplicaUnderTest(now time.Time, testedSta return true, "it is no longer actively tracked" } if lr.LastError != nil { - // LastError is set i.e. HealthCheck module cannot connect and the cached + // LastError is set i.e. LegacyHealthCheck module cannot connect and the cached // data for the replica might be outdated. return true, "it has LastError set i.e. is no longer correctly tracked" } diff --git a/go/vt/throttler/max_replication_lag_module_test.go b/go/vt/throttler/max_replication_lag_module_test.go index 100a76f7726..812cde0a0ad 100644 --- a/go/vt/throttler/max_replication_lag_module_test.go +++ b/go/vt/throttler/max_replication_lag_module_test.go @@ -223,7 +223,7 @@ func TestMaxReplicationLagModule_ReplicaUnderTest_LastErrorOrNotUp(t *testing.T) // r2 @ 75s, 0s lag, LastError set rError := lagRecord(sinceZero(75*time.Second), r2, 0) - rError.LastError = errors.New("HealthCheck reporting broken") + rError.LastError = errors.New("LegacyHealthCheck reporting broken") tf.m.replicaLagCache.add(rError) // r1 @ 110s, 0s lag @@ -945,13 +945,13 @@ func TestMaxReplicationLagModule_NoIncreaseIfMaxRateWasNotApproached(t *testing. } } -// lagRecord creates a fake record using a fake TabletStats object. +// lagRecord creates a fake record using a fake LegacyTabletStats object. func lagRecord(t time.Time, uid, lag uint32) replicationLagRecord { return replicationLagRecord{t, tabletStats(uid, lag)} } // tabletStats creates fake tablet health data. -func tabletStats(uid, lag uint32) discovery.TabletStats { +func tabletStats(uid, lag uint32) discovery.LegacyTabletStats { typ := topodatapb.TabletType_REPLICA if uid == rdonly1 || uid == rdonly2 { typ = topodatapb.TabletType_RDONLY @@ -963,7 +963,7 @@ func tabletStats(uid, lag uint32) discovery.TabletStats { Type: typ, PortMap: map[string]int32{"vt": int32(uid)}, } - return discovery.TabletStats{ + return discovery.LegacyTabletStats{ Tablet: tablet, Key: discovery.TabletToMapKey(tablet), Target: &querypb.Target{ diff --git a/go/vt/throttler/replication_lag_cache.go b/go/vt/throttler/replication_lag_cache.go index 5e14f5b86b6..9ab61eb81a8 100644 --- a/go/vt/throttler/replication_lag_cache.go +++ b/go/vt/throttler/replication_lag_cache.go @@ -27,11 +27,11 @@ import ( // replicationlagRecord entries. type replicationLagCache struct { // entries maps from the replica to its history. - // The map key is replicationLagRecord.TabletStats.Key. + // The map key is replicationLagRecord.LegacyTabletStats.Key. entries map[string]*replicationLagHistory // slowReplicas is a set of slow replicas. - // The map key is replicationLagRecord.TabletStats.Key. + // The map key is replicationLagRecord.LegacyTabletStats.Key. // This map will always be recomputed by sortByLag() and must not be modified // from other methods. slowReplicas map[string]bool @@ -43,7 +43,7 @@ type replicationLagCache struct { // becomes the new slowest replica. This set is used to detect such a chain. // The set will be cleared if ignoreSlowReplica() returns false. // - // The map key is replicationLagRecord.TabletStats.Key. + // The map key is replicationLagRecord.LegacyTabletStats.Key. // If an entry is deleted from "entries", it must be deleted here as well. ignoredSlowReplicasInARow map[string]bool @@ -76,7 +76,7 @@ func (c *replicationLagCache) add(r replicationLagRecord) { entry.add(r) } -// latest returns the current lag record for the given TabletStats.Key string. +// latest returns the current lag record for the given LegacyTabletStats.Key string. // A zero record is returned if there is no latest entry. func (c *replicationLagCache) latest(key string) replicationLagRecord { entry, ok := c.entries[key] @@ -114,7 +114,7 @@ func (c *replicationLagCache) sortByLag(ignoreNSlowestReplicas int, minimumRepli for _, v := range c.entries { record := v.latest() if int64(record.Stats.SecondsBehindMaster) >= minimumReplicationLag { - list = append(list, record.TabletStats) + list = append(list, record.LegacyTabletStats) i++ } } @@ -126,9 +126,9 @@ func (c *replicationLagCache) sortByLag(ignoreNSlowestReplicas int, minimumRepli } } -// byLagAndTabletUID is a slice of discovery.TabletStats elements that +// byLagAndTabletUID is a slice of discovery.LegacyTabletStats elements that // implements sort.Interface to sort by replication lag and tablet Uid. -type byLagAndTabletUID []discovery.TabletStats +type byLagAndTabletUID []discovery.LegacyTabletStats func (a byLagAndTabletUID) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byLagAndTabletUID) Len() int { return len(a) } @@ -140,7 +140,7 @@ func (a byLagAndTabletUID) Less(i, j int) bool { // ignoreSlowReplica returns true if the MaxReplicationLagModule should ignore // this slow replica. -// "key" refers to ReplicationLagRecord.TabletStats.Key. +// "key" refers to ReplicationLagRecord.LegacyTabletStats.Key. func (c *replicationLagCache) ignoreSlowReplica(key string) bool { if len(c.slowReplicas) == 0 { // No slow replicas at all. @@ -169,7 +169,7 @@ func (c *replicationLagCache) ignoreSlowReplica(key string) bool { } // isIgnored returns true if the given replica is a slow, ignored replica. -// "key" refers to ReplicationLagRecord.TabletStats.Key. +// "key" refers to ReplicationLagRecord.LegacyTabletStats.Key. // Note: Unlike ignoreSlowReplica(key), this method does not update the count // how many replicas in a row have been ignored. Instead, it's meant to find out // when a replica is ignored and therefore the module should not wait for it. diff --git a/go/vt/throttler/replication_lag_record.go b/go/vt/throttler/replication_lag_record.go index bd233ec8803..b0dff1e0a27 100644 --- a/go/vt/throttler/replication_lag_record.go +++ b/go/vt/throttler/replication_lag_record.go @@ -23,13 +23,13 @@ import ( ) // replicationLagRecord stores the tablet health data for a given point in time. -// This data is obtained via the HealthCheck module. +// This data is obtained via the LegacyHealthCheck module. type replicationLagRecord struct { // time is the time at which "value" was observed. time time.Time - // TabletStats holds a copy of the current health data of the tablet. - discovery.TabletStats + // LegacyTabletStats holds a copy of the current health data of the tablet. + discovery.LegacyTabletStats } func (r replicationLagRecord) isZero() bool { diff --git a/go/vt/throttler/throttler.go b/go/vt/throttler/throttler.go index 7de6ca65fdc..ea0096bc537 100644 --- a/go/vt/throttler/throttler.go +++ b/go/vt/throttler/throttler.go @@ -295,7 +295,7 @@ func (t *Throttler) SetMaxRate(rate int64) { // RecordReplicationLag must be called by users to report the "ts" tablet health // data observed at "time". // Note: After Close() is called, this method must not be called anymore. -func (t *Throttler) RecordReplicationLag(time time.Time, ts *discovery.TabletStats) { +func (t *Throttler) RecordReplicationLag(time time.Time, ts *discovery.LegacyTabletStats) { t.maxReplicationLagModule.RecordReplicationLag(time, ts) } diff --git a/go/vt/vtctld/realtime_status.go b/go/vt/vtctld/realtime_status.go index c3e04dfcb0c..cd0a6f21bf9 100644 --- a/go/vt/vtctld/realtime_status.go +++ b/go/vt/vtctld/realtime_status.go @@ -28,13 +28,13 @@ import ( // realtimeStats holds the objects needed to obtain realtime health stats of tablets. type realtimeStats struct { - healthCheck discovery.HealthCheck + healthCheck discovery.LegacyHealthCheck *tabletStatsCache cellWatchers []*discovery.TopologyWatcher } func newRealtimeStats(ts *topo.Server) (*realtimeStats, error) { - hc := discovery.NewHealthCheck(*vtctl.HealthcheckRetryDelay, *vtctl.HealthCheckTimeout) + hc := discovery.NewLegacyHealthCheck(*vtctl.HealthcheckRetryDelay, *vtctl.HealthCheckTimeout) tabletStatsCache := newTabletStatsCache() // sendDownEvents is set to true here, as we want to receive // Up=False events for a tablet. diff --git a/go/vt/vtctld/realtime_status_test.go b/go/vt/vtctld/realtime_status_test.go index 4df601a955e..4186ada084e 100644 --- a/go/vt/vtctld/realtime_status_test.go +++ b/go/vt/vtctld/realtime_status_test.go @@ -38,7 +38,7 @@ import ( ) // TestRealtimeStatsWithQueryService uses fakeTablets and the fakeQueryService to -// copy the environment needed for the HealthCheck object. +// copy the environment needed for the LegacyHealthCheck object. func TestRealtimeStatsWithQueryService(t *testing.T) { // Set up testing keyspace with 2 tablets within 2 cells. keyspace := "ks" @@ -113,7 +113,7 @@ func TestRealtimeStatsWithQueryService(t *testing.T) { } } -// checkStats ensures that the HealthCheck object received an update and passed +// checkStats ensures that the LegacyHealthCheck object received an update and passed // that information to the correct tablet. func checkStats(realtimeStats *realtimeStats, tablet *testlib.FakeTablet, want *querypb.RealtimeStats) error { deadline := time.Now().Add(time.Second * 5) @@ -122,7 +122,7 @@ func checkStats(realtimeStats *realtimeStats, tablet *testlib.FakeTablet, want * if err != nil { continue } - if result.DeepEqual(&discovery.TabletStats{}) { + if result.DeepEqual(&discovery.LegacyTabletStats{}) { continue } got := result.Stats @@ -134,7 +134,7 @@ func checkStats(realtimeStats *realtimeStats, tablet *testlib.FakeTablet, want * return fmt.Errorf("timeout error when getting tabletStatuses") } -// newRealtimeStatsForTesting creates a new realtimeStats object without creating a HealthCheck object. +// newRealtimeStatsForTesting creates a new realtimeStats object without creating a LegacyHealthCheck object. func newRealtimeStatsForTesting() *realtimeStats { tabletStatsCache := newTabletStatsCache() return &realtimeStats{ diff --git a/go/vt/vtctld/tablet_stats_cache.go b/go/vt/vtctld/tablet_stats_cache.go index 9e7e0fa5dbd..88719d14198 100644 --- a/go/vt/vtctld/tablet_stats_cache.go +++ b/go/vt/vtctld/tablet_stats_cache.go @@ -66,7 +66,7 @@ type heatmap struct { YGridLines []float64 } -type byTabletUID []*discovery.TabletStats +type byTabletUID []*discovery.LegacyTabletStats func (a byTabletUID) Len() int { return len(a) } func (a byTabletUID) Swap(i, j int) { a[i], a[j] = a[j], a[i] } @@ -78,18 +78,18 @@ var availableTabletTypes = []topodatapb.TabletType{topodatapb.TabletType_MASTER, // tabletStatsCache holds the most recent status update received for // each tablet. The tablets are indexed by uid, so it is different -// than discovery.TabletStatsCache. +// than discovery.LegacyTabletStatsCache. type tabletStatsCache struct { // mu guards access to the fields below. mu sync.Mutex - // statuses keeps a map of TabletStats. + // statuses keeps a map of LegacyTabletStats. // The first key is the keyspace, the second key is the shard, // the third key is the cell, the last key is the tabletType. // The keys are strings to allow exposing this map as a JSON object in api.go. - statuses map[string]map[string]map[string]map[topodatapb.TabletType][]*discovery.TabletStats + statuses map[string]map[string]map[string]map[topodatapb.TabletType][]*discovery.LegacyTabletStats // statusesByAlias is a copy of statuses and will be updated simultaneously. // The first key is the string representation of the tablet alias. - statusesByAlias map[string]*discovery.TabletStats + statusesByAlias map[string]*discovery.LegacyTabletStats } type topologyInfo struct { @@ -100,14 +100,14 @@ type topologyInfo struct { func newTabletStatsCache() *tabletStatsCache { return &tabletStatsCache{ - statuses: make(map[string]map[string]map[string]map[topodatapb.TabletType][]*discovery.TabletStats), - statusesByAlias: make(map[string]*discovery.TabletStats), + statuses: make(map[string]map[string]map[string]map[topodatapb.TabletType][]*discovery.LegacyTabletStats), + statusesByAlias: make(map[string]*discovery.LegacyTabletStats), } } -// StatsUpdate is part of the discovery.HealthCheckStatsListener interface. -// Upon receiving a new TabletStats, it updates the two maps in tablet_stats_cache. -func (c *tabletStatsCache) StatsUpdate(stats *discovery.TabletStats) { +// StatsUpdate is part of the discovery.LegacyHealthCheckStatsListener interface. +// Upon receiving a new LegacyTabletStats, it updates the two maps in tablet_stats_cache. +func (c *tabletStatsCache) StatsUpdate(stats *discovery.LegacyTabletStats) { c.mu.Lock() defer c.mu.Unlock() @@ -133,25 +133,25 @@ func (c *tabletStatsCache) StatsUpdate(stats *discovery.TabletStats) { // Tablet isn't tracked yet so just add it. _, ok := c.statuses[keyspace] if !ok { - shards := make(map[string]map[string]map[topodatapb.TabletType][]*discovery.TabletStats) + shards := make(map[string]map[string]map[topodatapb.TabletType][]*discovery.LegacyTabletStats) c.statuses[keyspace] = shards } _, ok = c.statuses[keyspace][shard] if !ok { - cells := make(map[string]map[topodatapb.TabletType][]*discovery.TabletStats) + cells := make(map[string]map[topodatapb.TabletType][]*discovery.LegacyTabletStats) c.statuses[keyspace][shard] = cells } _, ok = c.statuses[keyspace][shard][cell] if !ok { - types := make(map[topodatapb.TabletType][]*discovery.TabletStats) + types := make(map[topodatapb.TabletType][]*discovery.LegacyTabletStats) c.statuses[keyspace][shard][cell] = types } _, ok = c.statuses[keyspace][shard][cell][tabletType] if !ok { - tablets := make([]*discovery.TabletStats, 0) + tablets := make([]*discovery.LegacyTabletStats, 0) c.statuses[keyspace][shard][cell][tabletType] = tablets } @@ -165,13 +165,13 @@ func (c *tabletStatsCache) StatsUpdate(stats *discovery.TabletStats) { *ts = *stats } -func tabletToMapKey(stats *discovery.TabletStats) string { +func tabletToMapKey(stats *discovery.LegacyTabletStats) string { return stats.Tablet.Alias.String() } // remove takes in an array and returns it with the specified element removed // (leaves the array unchanged if element isn't in the array). -func remove(tablets []*discovery.TabletStats, tabletAlias *topodatapb.TabletAlias) []*discovery.TabletStats { +func remove(tablets []*discovery.LegacyTabletStats, tabletAlias *topodatapb.TabletAlias) []*discovery.LegacyTabletStats { filteredTablets := tablets[:0] for _, tablet := range tablets { if !topoproto.TabletAliasEqual(tablet.Tablet.Alias, tabletAlias) { @@ -312,7 +312,7 @@ func (c *tabletStatsCache) heatmapData(selectedKeyspace, selectedCell, selectedT defer c.mu.Unlock() // Get the metric data. - var metricFunc func(stats *discovery.TabletStats) float64 + var metricFunc func(stats *discovery.LegacyTabletStats) float64 switch selectedMetric { case "lag": metricFunc = replicationLag @@ -399,7 +399,7 @@ func (c *tabletStatsCache) heatmapData(selectedKeyspace, selectedCell, selectedT return heatmaps, nil } -func (c *tabletStatsCache) unaggregatedData(keyspace, cell, selectedType string, metricFunc func(stats *discovery.TabletStats) float64) ([][]float64, [][]*topodatapb.TabletAlias, yLabel) { +func (c *tabletStatsCache) unaggregatedData(keyspace, cell, selectedType string, metricFunc func(stats *discovery.LegacyTabletStats) float64) ([][]float64, [][]*topodatapb.TabletAlias, yLabel) { // This loop goes through every nested label (in this case, tablet type). var cellData [][]float64 var cellAliases [][]*topodatapb.TabletAlias @@ -459,7 +459,7 @@ func (c *tabletStatsCache) unaggregatedData(keyspace, cell, selectedType string, // aggregatedData gets heatmapData by taking the average of the metric value of all tablets within the keyspace and cell of the // specified type (or from all types if 'all' was selected). -func (c *tabletStatsCache) aggregatedData(keyspace, cell, selectedType, selectedMetric string, metricFunc func(stats *discovery.TabletStats) float64) ([][]float64, [][]*topodatapb.TabletAlias, yLabel) { +func (c *tabletStatsCache) aggregatedData(keyspace, cell, selectedType, selectedMetric string, metricFunc func(stats *discovery.LegacyTabletStats) float64) ([][]float64, [][]*topodatapb.TabletAlias, yLabel) { shards := c.shards(keyspace) tabletTypes := c.tabletTypesLocked(keyspace, cell, selectedType) @@ -508,18 +508,18 @@ func (c *tabletStatsCache) aggregatedData(keyspace, cell, selectedType, selected return cellData, nil, cellLabel } -func (c *tabletStatsCache) tabletStats(tabletAlias *topodatapb.TabletAlias) (discovery.TabletStats, error) { +func (c *tabletStatsCache) tabletStats(tabletAlias *topodatapb.TabletAlias) (discovery.LegacyTabletStats, error) { c.mu.Lock() defer c.mu.Unlock() ts, ok := c.statusesByAlias[tabletAlias.String()] if !ok { - return discovery.TabletStats{}, fmt.Errorf("could not find tablet: %v", tabletAlias) + return discovery.LegacyTabletStats{}, fmt.Errorf("could not find tablet: %v", tabletAlias) } return *ts, nil } -func health(stat *discovery.TabletStats) float64 { +func health(stat *discovery.LegacyTabletStats) float64 { // The tablet is unhealthy if there is an health error. if stat.Stats.HealthError != "" { return tabletUnhealthy @@ -548,13 +548,13 @@ func health(stat *discovery.TabletStats) float64 { return tabletHealthy } -func replicationLag(stat *discovery.TabletStats) float64 { +func replicationLag(stat *discovery.LegacyTabletStats) float64 { return float64(stat.Stats.SecondsBehindMaster) } -func qps(stat *discovery.TabletStats) float64 { +func qps(stat *discovery.LegacyTabletStats) float64 { return stat.Stats.Qps } // compile-time interface check -var _ discovery.HealthCheckStatsListener = (*tabletStatsCache)(nil) +var _ discovery.LegacyHealthCheckStatsListener = (*tabletStatsCache)(nil) diff --git a/go/vt/vtctld/tablet_stats_cache_test.go b/go/vt/vtctld/tablet_stats_cache_test.go index 781d0c98f77..eb4d4415456 100644 --- a/go/vt/vtctld/tablet_stats_cache_test.go +++ b/go/vt/vtctld/tablet_stats_cache_test.go @@ -342,7 +342,7 @@ func TestTabletStats(t *testing.T) { tabletStatsCache.StatsUpdate(ts1) tabletStatsCache.StatsUpdate(ts2) - // Test 1: tablet1 and tablet2 are updated with the stats received by the HealthCheck module. + // Test 1: tablet1 and tablet2 are updated with the stats received by the LegacyHealthCheck module. got1, err := tabletStatsCache.tabletStats(ts1.Tablet.Alias) want1 := ts1 if err != nil || !got1.DeepEqual(want1) { @@ -426,8 +426,8 @@ func TestTopologyInfo(t *testing.T) { } } -// tabletStats will create a discovery.TabletStats object. -func tabletStats(keyspace, cell, shard string, tabletType topodatapb.TabletType, uid uint32) *discovery.TabletStats { +// tabletStats will create a discovery.LegacyTabletStats object. +func tabletStats(keyspace, cell, shard string, tabletType topodatapb.TabletType, uid uint32) *discovery.LegacyTabletStats { target := &querypb.Target{ Keyspace: keyspace, Shard: shard, @@ -445,7 +445,7 @@ func tabletStats(keyspace, cell, shard string, tabletType topodatapb.TabletType, // uid is used for SecondsBehindMaster to give it a unique value. SecondsBehindMaster: uid, } - stats := &discovery.TabletStats{ + stats := &discovery.LegacyTabletStats{ Tablet: tablet, Target: target, Up: true, diff --git a/go/vt/vtexplain/vtexplain_vtgate.go b/go/vt/vtexplain/vtexplain_vtgate.go index 95259873f67..bee40e18e2d 100644 --- a/go/vt/vtexplain/vtexplain_vtgate.go +++ b/go/vt/vtexplain/vtexplain_vtgate.go @@ -70,7 +70,7 @@ func initVtgateExecutor(vSchemaStr string, opts *Options) error { return nil } -func newFakeResolver(opts *Options, hc discovery.HealthCheck, serv srvtopo.Server, cell string) *vtgate.Resolver { +func newFakeResolver(opts *Options, hc discovery.LegacyHealthCheck, serv srvtopo.Server, cell string) *vtgate.Resolver { ctx := context.Background() gw := vtgate.GatewayCreator()(ctx, hc, serv, cell, 3) gw.WaitForTablets(ctx, []topodatapb.TabletType{topodatapb.TabletType_REPLICA}) @@ -80,7 +80,7 @@ func newFakeResolver(opts *Options, hc discovery.HealthCheck, serv srvtopo.Serve txMode = vtgatepb.TransactionMode_TWOPC } tc := vtgate.NewTxConn(gw, txMode) - sc := vtgate.NewScatterConn("", tc, gw, hc) + sc := vtgate.LegacyNewScatterConn("", tc, gw, hc) srvResolver := srvtopo.NewResolver(serv, gw, cell) return vtgate.NewResolver(srvResolver, serv, cell, sc) } diff --git a/go/vt/vtgate/api.go b/go/vt/vtgate/api.go index c2a6f36e747..05f5c85d04d 100644 --- a/go/vt/vtgate/api.go +++ b/go/vt/vtgate/api.go @@ -88,7 +88,7 @@ func getItemPath(url string) string { return parts[1] } -func initAPI(ctx context.Context, hc discovery.HealthCheck) { +func initAPI(ctx context.Context, hc discovery.LegacyHealthCheck) { // Healthcheck real time status per (cell, keyspace, tablet type, metric). handleCollection("health-check", func(r *http.Request) (interface{}, error) { cacheStatus := hc.CacheStatus() @@ -110,7 +110,7 @@ func initAPI(ctx context.Context, hc discovery.HealthCheck) { switch collectionFilter { case "cell": { - filteredStatus := make(discovery.TabletsCacheStatusList, 0) + filteredStatus := make(discovery.LegacyTabletsCacheStatusList, 0) for _, tabletCacheStatus := range cacheStatus { if tabletCacheStatus.Cell == value { filteredStatus = append(filteredStatus, tabletCacheStatus) @@ -120,7 +120,7 @@ func initAPI(ctx context.Context, hc discovery.HealthCheck) { } case "keyspace": { - filteredStatus := make(discovery.TabletsCacheStatusList, 0) + filteredStatus := make(discovery.LegacyTabletsCacheStatusList, 0) for _, tabletCacheStatus := range cacheStatus { if tabletCacheStatus.Target.Keyspace == value { filteredStatus = append(filteredStatus, tabletCacheStatus) diff --git a/go/vt/vtgate/buffer/buffer.go b/go/vt/vtgate/buffer/buffer.go index 3cfe150930b..08fcb05b52d 100644 --- a/go/vt/vtgate/buffer/buffer.go +++ b/go/vt/vtgate/buffer/buffer.go @@ -88,7 +88,7 @@ type Buffer struct { // In particular, it is used to serialize the following Go routines: // - 1. Requests which may buffer (RLock, can be run in parallel) // - 2. Request which starts buffering (based on the seen error) - // - 3. HealthCheck listener ("StatsUpdate") which stops buffering + // - 3. LegacyHealthCheck listener ("StatsUpdate") which stops buffering // - 4. Timer which may stop buffering after -buffer_max_failover_duration mu sync.RWMutex // buffers holds a shardBuffer object per shard, even if no failover is in @@ -215,10 +215,10 @@ func (b *Buffer) WaitForFailoverEnd(ctx context.Context, keyspace, shard string, // StatsUpdate keeps track of the "tablet_externally_reparented_timestamp" of // each master. This way we can detect the end of a failover. -// It is part of the discovery.HealthCheckStatsListener interface. -func (b *Buffer) StatsUpdate(ts *discovery.TabletStats) { +// It is part of the discovery.LegacyHealthCheckStatsListener interface. +func (b *Buffer) StatsUpdate(ts *discovery.LegacyTabletStats) { if ts.Target.TabletType != topodatapb.TabletType_MASTER { - panic(fmt.Sprintf("BUG: non MASTER TabletStats object must not be forwarded: %#v", ts)) + panic(fmt.Sprintf("BUG: non MASTER LegacyTabletStats object must not be forwarded: %#v", ts)) } timestamp := ts.TabletExternallyReparentedTimestamp diff --git a/go/vt/vtgate/buffer/buffer_test.go b/go/vt/vtgate/buffer/buffer_test.go index 59a2d79ed95..b2c1c7f9c2f 100644 --- a/go/vt/vtgate/buffer/buffer_test.go +++ b/go/vt/vtgate/buffer/buffer_test.go @@ -93,7 +93,7 @@ func TestBuffer(t *testing.T) { // after this. If the TabletExternallyReparented RPC is called regularly by // an external failover tool, the timestamp will be increased (even though // the master did not change.) - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: oldMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: now.Unix(), @@ -123,7 +123,7 @@ func TestBuffer(t *testing.T) { // Mimic the failover end. now = now.Add(1 * time.Second) - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: newMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: now.Unix(), @@ -184,7 +184,7 @@ func TestBuffer(t *testing.T) { t.Fatalf("buffering start was not tracked: got = %v, want = %v", got, want) } // Stop buffering. - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: oldMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: now.Unix(), @@ -321,7 +321,7 @@ func TestDryRun(t *testing.T) { } // End of failover is tracked as well. - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: newMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: 1, // Use any value > 0. @@ -373,7 +373,7 @@ func TestLastReparentTooRecent_BufferingSkipped(t *testing.T) { // Simulate that the old master notified us about its reparented timestamp // very recently (time.Now()). // vtgate should see this immediately after the start. - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: oldMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: now.Unix(), @@ -382,7 +382,7 @@ func TestLastReparentTooRecent_BufferingSkipped(t *testing.T) { // Failover to new master. Its end is detected faster than the beginning. // Do not start buffering. now = now.Add(1 * time.Second) - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: newMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: now.Unix(), @@ -417,7 +417,7 @@ func TestLastReparentTooRecent_Buffering(t *testing.T) { // Simulate that the old master notified us about its reparented timestamp // very recently (time.Now()). // vtgate should see this immediately after the start. - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: oldMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: now.Unix(), @@ -426,7 +426,7 @@ func TestLastReparentTooRecent_Buffering(t *testing.T) { // Failover to new master. Do not issue any requests before or after i.e. // there was 0 QPS traffic and no buffering was started. now = now.Add(1 * time.Second) - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: newMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: now.Unix(), @@ -441,7 +441,7 @@ func TestLastReparentTooRecent_Buffering(t *testing.T) { t.Fatal(err) } // And then the failover end. - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: newMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: now.Unix(), @@ -480,7 +480,7 @@ func TestPassthroughDuringDrain(t *testing.T) { } // Stop buffering and trigger drain. - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: newMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: 1, // Use any value > 0. @@ -595,7 +595,7 @@ func testRequestCanceled(t *testing.T, explicitEnd bool) { } if explicitEnd { - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: newMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: 1, // Use any value > 0. @@ -614,7 +614,7 @@ func testRequestCanceled(t *testing.T, explicitEnd bool) { // If buffering stopped implicitly, the explicit signal will still happen // shortly after. In that case, the buffer should ignore it. if !explicitEnd { - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: newMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: 1, // Use any value > 0. @@ -660,7 +660,7 @@ func TestEviction(t *testing.T) { } // End of failover. Stop buffering. - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: newMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: 1, // Use any value > 0. @@ -743,7 +743,7 @@ func TestEvictionNotPossible(t *testing.T) { } // End of failover. Stop buffering. - b.StatsUpdate(&discovery.TabletStats{ + b.StatsUpdate(&discovery.LegacyTabletStats{ Tablet: newMaster, Target: &querypb.Target{Keyspace: keyspace, Shard: shard, TabletType: topodatapb.TabletType_MASTER}, TabletExternallyReparentedTimestamp: 1, // Use any value > 0. diff --git a/go/vt/vtgate/discoverygateway.go b/go/vt/vtgate/discoverygateway.go index 474ec99d35f..118084313e0 100644 --- a/go/vt/vtgate/discoverygateway.go +++ b/go/vt/vtgate/discoverygateway.go @@ -67,8 +67,8 @@ func init() { type discoveryGateway struct { queryservice.QueryService - hc discovery.HealthCheck - tsc *discovery.TabletStatsCache + hc discovery.LegacyHealthCheck + tsc *discovery.LegacyTabletStatsCache srvTopoServer srvtopo.Server localCell string retryCount int @@ -87,11 +87,15 @@ type discoveryGateway struct { buffer *buffer.Buffer } -func createDiscoveryGateway(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cell string, retryCount int) Gateway { +func createDiscoveryGateway(ctx context.Context, hc discovery.LegacyHealthCheck, serv srvtopo.Server, cell string, retryCount int) Gateway { return NewDiscoveryGateway(ctx, hc, serv, cell, retryCount) } -func NewDiscoveryGateway(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cell string, retryCount int) *discoveryGateway { +// NewDiscoveryGateway creates a new discoveryGateway using the provided healthcheck and toposerver. +// cell is the cell where the gateway is located a.k.a localCell. +// This gateway can route to MASTER in any cell provided by the cells_to_watch command line argument. +// Other tablet type requests (REPLICA/RDONLY) are only routed to tablets in the same cell. +func NewDiscoveryGateway(ctx context.Context, hc discovery.LegacyHealthCheck, serv srvtopo.Server, cell string, retryCount int) *discoveryGateway { var topoServer *topo.Server if serv != nil { var err error @@ -112,8 +116,8 @@ func NewDiscoveryGateway(ctx context.Context, hc discovery.HealthCheck, serv srv buffer: buffer.New(), } - // Set listener which will update TabletStatsCache and MasterBuffer. - // We set sendDownEvents=true because it's required by TabletStatsCache. + // Set listener which will update LegacyTabletStatsCache and MasterBuffer. + // We set sendDownEvents=true because it's required by LegacyTabletStatsCache. hc.SetListener(dg, true /* sendDownEvents */) log.Infof("loading tablets for cells: %v", *cellsToWatch) @@ -181,9 +185,9 @@ func (dg *discoveryGateway) topologyWatcherChecksum() int64 { return checksum } -// StatsUpdate forwards HealthCheck updates to TabletStatsCache and MasterBuffer. -// It is part of the discovery.HealthCheckStatsListener interface. -func (dg *discoveryGateway) StatsUpdate(ts *discovery.TabletStats) { +// StatsUpdate forwards LegacyHealthCheck updates to LegacyTabletStatsCache and MasterBuffer. +// It is part of the discovery.LegacyHealthCheckStatsListener interface. +func (dg *discoveryGateway) StatsUpdate(ts *discovery.LegacyTabletStats) { dg.tsc.StatsUpdate(ts) if ts.Target.TabletType == topodatapb.TabletType_MASTER { @@ -290,7 +294,7 @@ func (dg *discoveryGateway) withRetry(ctx context.Context, target *querypb.Targe shuffleTablets(dg.localCell, tablets) // skip tablets we tried before - var ts *discovery.TabletStats + var ts *discovery.LegacyTabletStats for _, t := range tablets { if _, ok := invalidTablets[t.Key]; !ok { ts = &t @@ -327,7 +331,7 @@ func (dg *discoveryGateway) withRetry(ctx context.Context, target *querypb.Targe return NewShardError(err, target, tabletLastUsed) } -func shuffleTablets(cell string, tablets []discovery.TabletStats) { +func shuffleTablets(cell string, tablets []discovery.LegacyTabletStats) { sameCell, diffCell, sameCellMax := 0, 0, -1 length := len(tablets) @@ -365,7 +369,7 @@ func shuffleTablets(cell string, tablets []discovery.TabletStats) { } } -func nextTablet(cell string, tablets []discovery.TabletStats, offset, length int, sameCell bool) int { +func nextTablet(cell string, tablets []discovery.LegacyTabletStats, offset, length int, sameCell bool) int { for ; offset < length; offset++ { if (tablets[offset].Tablet.Alias.Cell == cell) == sameCell { return offset diff --git a/go/vt/vtgate/discoverygateway_test.go b/go/vt/vtgate/discoverygateway_test.go index ce7508f96e1..553a8bfa1f9 100644 --- a/go/vt/vtgate/discoverygateway_test.go +++ b/go/vt/vtgate/discoverygateway_test.go @@ -131,7 +131,7 @@ func TestDiscoveryGatewayGetTablets(t *testing.T) { } func TestShuffleTablets(t *testing.T) { - ts1 := discovery.TabletStats{ + ts1 := discovery.LegacyTabletStats{ Key: "t1", Tablet: topo.NewTablet(10, "cell1", "host1"), Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -140,7 +140,7 @@ func TestShuffleTablets(t *testing.T) { Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - ts2 := discovery.TabletStats{ + ts2 := discovery.LegacyTabletStats{ Key: "t2", Tablet: topo.NewTablet(10, "cell1", "host2"), Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -149,7 +149,7 @@ func TestShuffleTablets(t *testing.T) { Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - ts3 := discovery.TabletStats{ + ts3 := discovery.LegacyTabletStats{ Key: "t3", Tablet: topo.NewTablet(10, "cell2", "host3"), Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -158,7 +158,7 @@ func TestShuffleTablets(t *testing.T) { Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - ts4 := discovery.TabletStats{ + ts4 := discovery.LegacyTabletStats{ Key: "t4", Tablet: topo.NewTablet(10, "cell2", "host4"), Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -167,9 +167,9 @@ func TestShuffleTablets(t *testing.T) { Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - sameCellTablets := []discovery.TabletStats{ts1, ts2} - diffCellTablets := []discovery.TabletStats{ts3, ts4} - mixedTablets := []discovery.TabletStats{ts1, ts2, ts3, ts4} + sameCellTablets := []discovery.LegacyTabletStats{ts1, ts2} + diffCellTablets := []discovery.LegacyTabletStats{ts3, ts4} + mixedTablets := []discovery.LegacyTabletStats{ts1, ts2, ts3, ts4} // repeat shuffling 10 times and every time the same cell tablets should be in the front for i := 0; i < 10; i++ { shuffleTablets("cell1", sameCellTablets) diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 69eb0b7753a..852b47770ae 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -841,7 +841,7 @@ func (e *Executor) handleShow(ctx context.Context, safeSession *SafeSession, sql }, nil case "vitess_tablets": var rows [][]sqltypes.Value - stats := e.scatterConn.healthCheck.CacheStatus() + stats := e.scatterConn.GetHealthCheckCacheStatus() for _, s := range stats { for _, ts := range s.TabletsStats { state := "SERVING" diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index f8621cea6c0..a4e3e833c96 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -576,7 +576,7 @@ func testQueryLog(t *testing.T, logChan chan interface{}, method, stmtType, sql return logStats } -func newTestResolver(hc discovery.HealthCheck, serv srvtopo.Server, cell string) *Resolver { +func newTestResolver(hc discovery.LegacyHealthCheck, serv srvtopo.Server, cell string) *Resolver { sc := newTestScatterConn(hc, serv, cell) srvResolver := srvtopo.NewResolver(serv, sc.gateway, cell) return NewResolver(srvResolver, serv, cell, sc) diff --git a/go/vt/vtgate/gateway.go b/go/vt/vtgate/gateway.go index 8bf30833372..47d03f8bd50 100644 --- a/go/vt/vtgate/gateway.go +++ b/go/vt/vtgate/gateway.go @@ -71,7 +71,7 @@ type Gateway interface { } // Creator is the factory method which can create the actual gateway object. -type Creator func(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cell string, retryCount int) Gateway +type Creator func(ctx context.Context, hc discovery.LegacyHealthCheck, serv srvtopo.Server, cell string, retryCount int) Gateway var creators = make(map[string]Creator) diff --git a/go/vt/vtgate/grpc_discovery_test.go b/go/vt/vtgate/grpc_discovery_test.go index 03d22a810ad..61b12aa47e2 100644 --- a/go/vt/vtgate/grpc_discovery_test.go +++ b/go/vt/vtgate/grpc_discovery_test.go @@ -62,7 +62,7 @@ func TestGRPCDiscovery(t *testing.T) { // VTGate: create the discovery healthcheck, and the gateway. // Wait for the right tablets to be present. - hc := discovery.NewHealthCheck(10*time.Second, 2*time.Minute) + hc := discovery.NewLegacyHealthCheck(10*time.Second, 2*time.Minute) rs := srvtopo.NewResilientServer(ts, "TestGRPCDiscovery") dg := NewDiscoveryGateway(context.Background(), hc, rs, cell, 2) hc.AddTablet(&topodatapb.Tablet{ diff --git a/go/vt/vtgate/scatter_conn.go b/go/vt/vtgate/scatter_conn.go index d3657aab4c7..dec03a16d1c 100644 --- a/go/vt/vtgate/scatter_conn.go +++ b/go/vt/vtgate/scatter_conn.go @@ -48,7 +48,7 @@ type ScatterConn struct { tabletCallErrorCount *stats.CountersWithMultiLabels txConn *TxConn gateway Gateway - healthCheck discovery.HealthCheck + legacyHealthCheck discovery.LegacyHealthCheck } // shardActionFunc defines the contract for a shard action @@ -68,8 +68,8 @@ type shardActionFunc func(rs *srvtopo.ResolvedShard, i int) error // the results and errors for the caller. type shardActionTransactionFunc func(rs *srvtopo.ResolvedShard, i int, shouldBegin bool, transactionID int64) (int64, error) -// NewScatterConn creates a new ScatterConn. -func NewScatterConn(statsName string, txConn *TxConn, gw Gateway, hc discovery.HealthCheck) *ScatterConn { +// LegacyNewScatterConn creates a new ScatterConn. +func LegacyNewScatterConn(statsName string, txConn *TxConn, gw Gateway, hc discovery.LegacyHealthCheck) *ScatterConn { tabletCallErrorCountStatsName := "" if statsName != "" { tabletCallErrorCountStatsName = statsName + "ErrorCount" @@ -83,9 +83,9 @@ func NewScatterConn(statsName string, txConn *TxConn, gw Gateway, hc discovery.H tabletCallErrorCountStatsName, "Error count from tablet calls in scatter conns", []string{"Operation", "Keyspace", "ShardName", "DbType"}), - txConn: txConn, - gateway: gw, - healthCheck: hc, + txConn: txConn, + gateway: gw, + legacyHealthCheck: hc, } } @@ -419,6 +419,11 @@ func (stc *ScatterConn) GetGatewayCacheStatus() TabletCacheStatusList { return stc.gateway.CacheStatus() } +// GetHealthCheckCacheStatus returns a displayable version of the HealthCheck cache. +func (stc *ScatterConn) GetHealthCheckCacheStatus() discovery.LegacyTabletsCacheStatusList { + return stc.legacyHealthCheck.CacheStatus() +} + // multiGo performs the requested 'action' on the specified // shards in parallel. This does not handle any transaction state. // The action function must match the shardActionFunc2 signature. diff --git a/go/vt/vtgate/scatter_conn_test.go b/go/vt/vtgate/scatter_conn_test.go index ede85cc5cfc..3e905ee999a 100644 --- a/go/vt/vtgate/scatter_conn_test.go +++ b/go/vt/vtgate/scatter_conn_test.go @@ -648,11 +648,11 @@ func TestAppendResult(t *testing.T) { } } -func newTestScatterConn(hc discovery.HealthCheck, serv srvtopo.Server, cell string) *ScatterConn { +func newTestScatterConn(hc discovery.LegacyHealthCheck, serv srvtopo.Server, cell string) *ScatterConn { // The topo.Server is used to start watching the cells described // in '-cells_to_watch' command line parameter, which is // empty by default. So it's unused in this test, set to nil. gw := GatewayCreator()(context.Background(), hc, serv, cell, 3) tc := NewTxConn(gw, vtgatepb.TransactionMode_TWOPC) - return NewScatterConn("", tc, gw, hc) + return LegacyNewScatterConn("", tc, gw, hc) } diff --git a/go/vt/vtgate/vstream_manager_test.go b/go/vt/vtgate/vstream_manager_test.go index 94aeea2a1ed..6779a9b889c 100644 --- a/go/vt/vtgate/vstream_manager_test.go +++ b/go/vt/vtgate/vstream_manager_test.go @@ -872,7 +872,7 @@ func TestResolveVStreamParams(t *testing.T) { } } -func newTestVStreamManager(hc discovery.HealthCheck, serv srvtopo.Server, cell string) *vstreamManager { +func newTestVStreamManager(hc discovery.LegacyHealthCheck, serv srvtopo.Server, cell string) *vstreamManager { gw := NewDiscoveryGateway(context.Background(), hc, serv, cell, 3) srvResolver := srvtopo.NewResolver(serv, gw, cell) return newVStreamManager(srvResolver, serv, cell) diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index 0a92c6dbf95..c93ce8b6c9f 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -120,91 +120,6 @@ type RegisterVTGate func(vtgateservice.VTGateService) // RegisterVTGates stores register funcs for VTGate server. var RegisterVTGates []RegisterVTGate -// Init initializes VTGate server. -func Init(ctx context.Context, hc discovery.HealthCheck, serv srvtopo.Server, cell string, retryCount int, tabletTypesToWait []topodatapb.TabletType) *VTGate { - if rpcVTGate != nil { - log.Fatalf("VTGate already initialized") - } - - // vschemaCounters needs to be initialized before planner to - // catch the initial load stats. - vschemaCounters = stats.NewCountersWithSingleLabel("VtgateVSchemaCounts", "Vtgate vschema counts", "changes") - - // Build objects from low to high level. - // Start with the gateway. If we can't reach the topology service, - // we can't go on much further, so we log.Fatal out. - gw := GatewayCreator()(ctx, hc, serv, cell, retryCount) - gw.RegisterStats() - if err := WaitForTablets(gw, tabletTypesToWait); err != nil { - log.Fatalf("gateway.WaitForTablets failed: %v", err) - } - - // If we want to filter keyspaces replace the srvtopo.Server with a - // filtering server - if len(KeyspacesToWatch) > 0 { - log.Infof("Keyspace filtering enabled, selecting %v", KeyspacesToWatch) - var err error - serv, err = srvtopo.NewKeyspaceFilteringServer(serv, KeyspacesToWatch) - if err != nil { - log.Fatalf("Unable to construct SrvTopo server: %v", err.Error()) - } - } - - tc := NewTxConn(gw, getTxMode()) - // ScatterConn depends on TxConn to perform forced rollbacks. - sc := NewScatterConn("VttabletCall", tc, gw, hc) - srvResolver := srvtopo.NewResolver(serv, gw, cell) - resolver := NewResolver(srvResolver, serv, cell, sc) - vsm := newVStreamManager(srvResolver, serv, cell) - - rpcVTGate = &VTGate{ - executor: NewExecutor(ctx, serv, cell, resolver, *normalizeQueries, *streamBufferSize, *queryPlanCacheSize), - resolver: resolver, - vsm: vsm, - txConn: tc, - gw: gw, - timings: stats.NewMultiTimings( - "VtgateApi", - "VtgateApi timings", - []string{"Operation", "Keyspace", "DbType"}), - rowsReturned: stats.NewCountersWithMultiLabels( - "VtgateApiRowsReturned", - "Rows returned through the VTgate API", - []string{"Operation", "Keyspace", "DbType"}), - - logExecute: logutil.NewThrottledLogger("Execute", 5*time.Second), - logStreamExecute: logutil.NewThrottledLogger("StreamExecute", 5*time.Second), - } - - errorCounts = stats.NewCountersWithMultiLabels("VtgateApiErrorCounts", "Vtgate API error counts per error type", []string{"Operation", "Keyspace", "DbType", "Code"}) - - _ = stats.NewRates("QPSByOperation", stats.CounterForDimension(rpcVTGate.timings, "Operation"), 15, 1*time.Minute) - _ = stats.NewRates("QPSByKeyspace", stats.CounterForDimension(rpcVTGate.timings, "Keyspace"), 15, 1*time.Minute) - _ = stats.NewRates("QPSByDbType", stats.CounterForDimension(rpcVTGate.timings, "DbType"), 15*60/5, 5*time.Second) - - _ = stats.NewRates("ErrorsByOperation", stats.CounterForDimension(errorCounts, "Operation"), 15, 1*time.Minute) - _ = stats.NewRates("ErrorsByKeyspace", stats.CounterForDimension(errorCounts, "Keyspace"), 15, 1*time.Minute) - _ = stats.NewRates("ErrorsByDbType", stats.CounterForDimension(errorCounts, "DbType"), 15, 1*time.Minute) - _ = stats.NewRates("ErrorsByCode", stats.CounterForDimension(errorCounts, "Code"), 15, 1*time.Minute) - - warnings = stats.NewCountersWithSingleLabel("VtGateWarnings", "Vtgate warnings", "type", "IgnoredSet", "ResultsExceeded") - - servenv.OnRun(func() { - for _, f := range RegisterVTGates { - f(rpcVTGate) - } - }) - rpcVTGate.registerDebugHealthHandler() - err := initQueryLogger(rpcVTGate) - if err != nil { - log.Fatalf("error initializing query logger: %v", err) - } - - initAPI(ctx, hc) - - return rpcVTGate -} - func (vtg *VTGate) registerDebugHealthHandler() { http.HandleFunc("/debug/health", func(w http.ResponseWriter, r *http.Request) { if err := acl.CheckAccessHTTP(r, acl.MONITORING); err != nil { @@ -451,3 +366,88 @@ func (vtg *VTGate) HandlePanic(err *error) { errorCounts.Add([]string{"Panic", "Unknown", "Unknown", vtrpcpb.Code_INTERNAL.String()}, 1) } } + +// LegacyInit initializes VTGate server with LegacyHealthCheck +func LegacyInit(ctx context.Context, hc discovery.LegacyHealthCheck, serv srvtopo.Server, cell string, retryCount int, tabletTypesToWait []topodatapb.TabletType) *VTGate { + if rpcVTGate != nil { + log.Fatalf("VTGate already initialized") + } + + // vschemaCounters needs to be initialized before planner to + // catch the initial load stats. + vschemaCounters = stats.NewCountersWithSingleLabel("VtgateVSchemaCounts", "Vtgate vschema counts", "changes") + + // Build objects from low to high level. + // Start with the gateway. If we can't reach the topology service, + // we can't go on much further, so we log.Fatal out. + gw := GatewayCreator()(ctx, hc, serv, cell, retryCount) + gw.RegisterStats() + if err := WaitForTablets(gw, tabletTypesToWait); err != nil { + log.Fatalf("gateway.WaitForTablets failed: %v", err) + } + + // If we want to filter keyspaces replace the srvtopo.Server with a + // filtering server + if len(KeyspacesToWatch) > 0 { + log.Infof("Keyspace filtering enabled, selecting %v", KeyspacesToWatch) + var err error + serv, err = srvtopo.NewKeyspaceFilteringServer(serv, KeyspacesToWatch) + if err != nil { + log.Fatalf("Unable to construct SrvTopo server: %v", err.Error()) + } + } + + tc := NewTxConn(gw, getTxMode()) + // ScatterConn depends on TxConn to perform forced rollbacks. + sc := LegacyNewScatterConn("VttabletCall", tc, gw, hc) + srvResolver := srvtopo.NewResolver(serv, gw, cell) + resolver := NewResolver(srvResolver, serv, cell, sc) + vsm := newVStreamManager(srvResolver, serv, cell) + + rpcVTGate = &VTGate{ + executor: NewExecutor(ctx, serv, cell, resolver, *normalizeQueries, *streamBufferSize, *queryPlanCacheSize), + resolver: resolver, + vsm: vsm, + txConn: tc, + gw: gw, + timings: stats.NewMultiTimings( + "VtgateApi", + "VtgateApi timings", + []string{"Operation", "Keyspace", "DbType"}), + rowsReturned: stats.NewCountersWithMultiLabels( + "VtgateApiRowsReturned", + "Rows returned through the VTgate API", + []string{"Operation", "Keyspace", "DbType"}), + + logExecute: logutil.NewThrottledLogger("Execute", 5*time.Second), + logStreamExecute: logutil.NewThrottledLogger("StreamExecute", 5*time.Second), + } + + errorCounts = stats.NewCountersWithMultiLabels("VtgateApiErrorCounts", "Vtgate API error counts per error type", []string{"Operation", "Keyspace", "DbType", "Code"}) + + _ = stats.NewRates("QPSByOperation", stats.CounterForDimension(rpcVTGate.timings, "Operation"), 15, 1*time.Minute) + _ = stats.NewRates("QPSByKeyspace", stats.CounterForDimension(rpcVTGate.timings, "Keyspace"), 15, 1*time.Minute) + _ = stats.NewRates("QPSByDbType", stats.CounterForDimension(rpcVTGate.timings, "DbType"), 15*60/5, 5*time.Second) + + _ = stats.NewRates("ErrorsByOperation", stats.CounterForDimension(errorCounts, "Operation"), 15, 1*time.Minute) + _ = stats.NewRates("ErrorsByKeyspace", stats.CounterForDimension(errorCounts, "Keyspace"), 15, 1*time.Minute) + _ = stats.NewRates("ErrorsByDbType", stats.CounterForDimension(errorCounts, "DbType"), 15, 1*time.Minute) + _ = stats.NewRates("ErrorsByCode", stats.CounterForDimension(errorCounts, "Code"), 15, 1*time.Minute) + + warnings = stats.NewCountersWithSingleLabel("VtGateWarnings", "Vtgate warnings", "type", "IgnoredSet", "ResultsExceeded") + + servenv.OnRun(func() { + for _, f := range RegisterVTGates { + f(rpcVTGate) + } + }) + rpcVTGate.registerDebugHealthHandler() + err := initQueryLogger(rpcVTGate) + if err != nil { + log.Fatalf("error initializing query logger: %v", err) + } + + initAPI(ctx, hc) + + return rpcVTGate +} diff --git a/go/vt/vtgate/vtgate_test.go b/go/vt/vtgate/vtgate_test.go index 72d30c25851..16f11b025ab 100644 --- a/go/vt/vtgate/vtgate_test.go +++ b/go/vt/vtgate/vtgate_test.go @@ -74,7 +74,7 @@ func init() { // The topo.Server is used to start watching the cells described // in '-cells_to_watch' command line parameter, which is // empty by default. So it's unused in this test, set to nil. - Init(context.Background(), hcVTGateTest, new(sandboxTopo), "aa", 10, nil) + LegacyInit(context.Background(), hcVTGateTest, new(sandboxTopo), "aa", 10, nil) *mysqlServerPort = 0 *mysqlAuthServerImpl = "none" diff --git a/go/vt/vttablet/tabletserver/txthrottler/mock_healthcheck_test.go b/go/vt/vttablet/tabletserver/txthrottler/mock_healthcheck_test.go index c29c63a40aa..ecded2d0bc6 100644 --- a/go/vt/vttablet/tabletserver/txthrottler/mock_healthcheck_test.go +++ b/go/vt/vttablet/tabletserver/txthrottler/mock_healthcheck_test.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: vitess.io/vitess/go/vt/discovery (interfaces: HealthCheck) +// Source: vitess.io/vitess/go/vt/discovery (interfaces: LegacyHealthCheck) // Package txthrottler is a generated GoMock package. package txthrottler @@ -13,7 +13,7 @@ import ( queryservice "vitess.io/vitess/go/vt/vttablet/queryservice" ) -// MockHealthCheck is a mock of HealthCheck interface +// MockHealthCheck is a mock of LegacyHealthCheck interface type MockHealthCheck struct { ctrl *gomock.Controller recorder *MockHealthCheckMockRecorder @@ -47,9 +47,9 @@ func (mr *MockHealthCheckMockRecorder) AddTablet(arg0, arg1 interface{}) *gomock } // CacheStatus mocks base method -func (m *MockHealthCheck) CacheStatus() discovery.TabletsCacheStatusList { +func (m *MockHealthCheck) CacheStatus() discovery.LegacyTabletsCacheStatusList { ret := m.ctrl.Call(m, "CacheStatus") - ret0, _ := ret[0].(discovery.TabletsCacheStatusList) + ret0, _ := ret[0].(discovery.LegacyTabletsCacheStatusList) return ret0 } @@ -113,7 +113,7 @@ func (mr *MockHealthCheckMockRecorder) ReplaceTablet(arg0, arg1, arg2 interface{ } // SetListener mocks base method -func (m *MockHealthCheck) SetListener(arg0 discovery.HealthCheckStatsListener, arg1 bool) { +func (m *MockHealthCheck) SetListener(arg0 discovery.LegacyHealthCheckStatsListener, arg1 bool) { m.ctrl.Call(m, "SetListener", arg0, arg1) } diff --git a/go/vt/vttablet/tabletserver/txthrottler/mock_throttler_test.go b/go/vt/vttablet/tabletserver/txthrottler/mock_throttler_test.go index 7604b9b1f43..b760b1a88ae 100644 --- a/go/vt/vttablet/tabletserver/txthrottler/mock_throttler_test.go +++ b/go/vt/vttablet/tabletserver/txthrottler/mock_throttler_test.go @@ -71,7 +71,7 @@ func (mr *MockThrottlerInterfaceMockRecorder) MaxRate() *gomock.Call { } // RecordReplicationLag mocks base method -func (m *MockThrottlerInterface) RecordReplicationLag(arg0 time.Time, arg1 *discovery.TabletStats) { +func (m *MockThrottlerInterface) RecordReplicationLag(arg0 time.Time, arg1 *discovery.LegacyTabletStats) { m.ctrl.Call(m, "RecordReplicationLag", arg0, arg1) } diff --git a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go index 10683619133..9178290a1c1 100644 --- a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go +++ b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go @@ -36,7 +36,7 @@ import ( // TxThrottler throttles transactions based on replication lag. // It's a thin wrapper around the throttler found in vitess/go/vt/throttler. -// It uses a discovery.HealthCheck to send replication-lag updates to the wrapped throttler. +// It uses a discovery.LegacyHealthCheck to send replication-lag updates to the wrapped throttler. // // Intended Usage: // // Assuming topoServer is a topo.Server variable pointing to a Vitess topology server. @@ -137,7 +137,7 @@ type ThrottlerInterface interface { Close() MaxRate() int64 SetMaxRate(rate int64) - RecordReplicationLag(time time.Time, ts *discovery.TabletStats) + RecordReplicationLag(time time.Time, ts *discovery.LegacyTabletStats) GetConfiguration() *throttlerdatapb.Configuration UpdateConfiguration(configuration *throttlerdatapb.Configuration, copyZeroValues bool) error ResetConfiguration() @@ -158,14 +158,14 @@ type txThrottlerState struct { throttleMu sync.Mutex throttler ThrottlerInterface - healthCheck discovery.HealthCheck + healthCheck discovery.LegacyHealthCheck topologyWatchers []TopologyWatcherInterface } // These vars store the functions used to create the topo server, healthcheck, // topology watchers and go/vt/throttler. These are provided here so that they can be overridden // in tests to generate mocks. -type healthCheckFactoryFunc func() discovery.HealthCheck +type healthCheckFactoryFunc func() discovery.LegacyHealthCheck type topologyWatcherFactoryFunc func(topoServer *topo.Server, tr discovery.TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) TopologyWatcherInterface type throttlerFactoryFunc func(name, unit string, threadCount int, maxRate, maxReplicationLag int64) (ThrottlerInterface, error) @@ -180,7 +180,7 @@ func init() { } func resetTxThrottlerFactories() { - healthCheckFactory = discovery.NewDefaultHealthCheck + healthCheckFactory = discovery.NewLegacyDefaultHealthCheck topologyWatcherFactory = func(topoServer *topo.Server, tr discovery.TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) TopologyWatcherInterface { return discovery.NewShardReplicationWatcher(context.Background(), topoServer, tr, cell, keyspace, shard, refreshInterval, topoReadConcurrency) } @@ -316,8 +316,8 @@ func (ts *txThrottlerState) deallocateResources() { ts.throttler = nil } -// StatsUpdate is part of the HealthCheckStatsListener interface. -func (ts *txThrottlerState) StatsUpdate(tabletStats *discovery.TabletStats) { +// StatsUpdate is part of the LegacyHealthCheckStatsListener interface. +func (ts *txThrottlerState) StatsUpdate(tabletStats *discovery.LegacyTabletStats) { // Ignore MASTER and RDONLY stats. // We currently do not monitor RDONLY tablets for replication lag. RDONLY tablets are not // candidates for becoming master during failover, and it's acceptable to serve somewhat diff --git a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go index bca23f8b48f..eb11c553c40 100644 --- a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go +++ b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go @@ -17,7 +17,7 @@ limitations under the License. package txthrottler // Commands to generate the mocks for this test. -//go:generate mockgen -destination mock_healthcheck_test.go -package txthrottler vitess.io/vitess/go/vt/discovery HealthCheck +//go:generate mockgen -destination mock_healthcheck_test.go -package txthrottler vitess.io/vitess/go/vt/discovery LegacyHealthCheck //go:generate mockgen -destination mock_throttler_test.go -package txthrottler vitess.io/vitess/go/vt/vttablet/tabletserver/txthrottler ThrottlerInterface //go:generate mockgen -destination mock_topology_watcher_test.go -package txthrottler vitess.io/vitess/go/vt/vttablet/tabletserver/txthrottler TopologyWatcherInterface @@ -56,15 +56,15 @@ func TestEnabledThrottler(t *testing.T) { ts := memorytopo.NewServer("cell1", "cell2") mockHealthCheck := NewMockHealthCheck(mockCtrl) - var hcListener discovery.HealthCheckStatsListener + var hcListener discovery.LegacyHealthCheckStatsListener hcCall1 := mockHealthCheck.EXPECT().SetListener(gomock.Any(), false /* sendDownEvents */) - hcCall1.Do(func(listener discovery.HealthCheckStatsListener, sendDownEvents bool) { + hcCall1.Do(func(listener discovery.LegacyHealthCheckStatsListener, sendDownEvents bool) { // Record the listener we're given. hcListener = listener }) hcCall2 := mockHealthCheck.EXPECT().Close() hcCall2.After(hcCall1) - healthCheckFactory = func() discovery.HealthCheck { return mockHealthCheck } + healthCheckFactory = func() discovery.LegacyHealthCheck { return mockHealthCheck } topologyWatcherFactory = func(topoServer *topo.Server, tr discovery.TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) TopologyWatcherInterface { if ts != topoServer { @@ -95,7 +95,7 @@ func TestEnabledThrottler(t *testing.T) { call0 := mockThrottler.EXPECT().UpdateConfiguration(gomock.Any(), true /* copyZeroValues */) call1 := mockThrottler.EXPECT().Throttle(0) call1.Return(0 * time.Second) - tabletStats := &discovery.TabletStats{ + tabletStats := &discovery.LegacyTabletStats{ Target: &querypb.Target{ TabletType: topodatapb.TabletType_REPLICA, }, @@ -124,7 +124,7 @@ func TestEnabledThrottler(t *testing.T) { t.Errorf("want: false, got: %v", result) } hcListener.StatsUpdate(tabletStats) - rdonlyTabletStats := &discovery.TabletStats{ + rdonlyTabletStats := &discovery.LegacyTabletStats{ Target: &querypb.Target{ TabletType: topodatapb.TabletType_RDONLY, }, diff --git a/go/vt/worker/executor.go b/go/vt/worker/executor.go index f0e14db21de..9e559552111 100644 --- a/go/vt/worker/executor.go +++ b/go/vt/worker/executor.go @@ -41,7 +41,7 @@ import ( // executor is also used for executing vreplication and RefreshState commands. type executor struct { wr *wrangler.Wrangler - tsc *discovery.TabletStatsCache + tsc *discovery.LegacyTabletStatsCache throttler *throttler.Throttler keyspace string shard string @@ -51,7 +51,7 @@ type executor struct { statsKey []string } -func newExecutor(wr *wrangler.Wrangler, tsc *discovery.TabletStatsCache, throttler *throttler.Throttler, keyspace, shard string, threadID int) *executor { +func newExecutor(wr *wrangler.Wrangler, tsc *discovery.LegacyTabletStatsCache, throttler *throttler.Throttler, keyspace, shard string, threadID int) *executor { return &executor{ wr: wr, tsc: tsc, @@ -113,7 +113,7 @@ func (e *executor) refreshState(ctx context.Context) error { // it fails due to a timeout or a retriable application error. // // executeFetchWithRetries will always get the current MASTER tablet from the -// TabletStatsCache instance. If no MASTER is available, it will keep retrying. +// LegacyTabletStatsCache instance. If no MASTER is available, it will keep retrying. func (e *executor) fetchWithRetries(ctx context.Context, action func(ctx context.Context, tablet *topodatapb.Tablet) error) error { retryDuration := *retryDuration // We should keep retrying up until the retryCtx runs out. @@ -122,10 +122,10 @@ func (e *executor) fetchWithRetries(ctx context.Context, action func(ctx context // Is this current attempt a retry of a previous attempt? isRetry := false for { - var master *discovery.TabletStats + var master *discovery.LegacyTabletStats var err error - // Get the current master from the TabletStatsCache. + // Get the current master from the LegacyTabletStatsCache. masters := e.tsc.GetHealthyTabletStats(e.keyspace, e.shard, topodatapb.TabletType_MASTER) if len(masters) == 0 { e.wr.Logger().Warningf("ExecuteFetch failed for keyspace/shard %v/%v because no MASTER is available; will retry until there is MASTER again", e.keyspace, e.shard) @@ -185,7 +185,7 @@ func (e *executor) fetchWithRetries(ctx context.Context, action func(ctx context } return vterrors.Wrapf(err, "interrupted while trying to run a command on tablet %v", tabletString) case <-time.After(*executeFetchRetryTime): - // Retry 30s after the failure using the current master seen by the HealthCheck. + // Retry 30s after the failure using the current master seen by the LegacyHealthCheck. } isRetry = true } @@ -194,7 +194,7 @@ func (e *executor) fetchWithRetries(ctx context.Context, action func(ctx context // checkError returns true if the error can be ignored and the command // succeeded, false if the error is retryable and a non-nil error if the // command must not be retried. -func (e *executor) checkError(ctx context.Context, err error, isRetry bool, master *discovery.TabletStats) (bool, error) { +func (e *executor) checkError(ctx context.Context, err error, isRetry bool, master *discovery.LegacyTabletStats) (bool, error) { tabletString := fmt.Sprintf("%v (%v/%v)", topoproto.TabletAliasString(master.Tablet.Alias), e.keyspace, e.shard) // first see if it was a context timeout. diff --git a/go/vt/worker/legacy_split_clone.go b/go/vt/worker/legacy_split_clone.go index dd9e568e47a..46de0ca3d16 100644 --- a/go/vt/worker/legacy_split_clone.go +++ b/go/vt/worker/legacy_split_clone.go @@ -76,8 +76,8 @@ type LegacySplitCloneWorker struct { sourceTablets []*topodatapb.Tablet // healthCheck tracks the health of all MASTER and REPLICA tablets. // It must be closed at the end of the command. - healthCheck discovery.HealthCheck - tsc *discovery.TabletStatsCache + healthCheck discovery.LegacyHealthCheck + tsc *discovery.LegacyTabletStatsCache // destinationShardWatchers contains a TopologyWatcher for each destination // shard. It updates the list of tablets in the healthcheck if replicas are // added/removed. @@ -222,7 +222,7 @@ func (scw *LegacySplitCloneWorker) Run(ctx context.Context) error { } if scw.healthCheck != nil { if err := scw.healthCheck.Close(); err != nil { - scw.wr.Logger().Errorf2(err, "HealthCheck.Close() failed") + scw.wr.Logger().Errorf2(err, "LegacyHealthCheck.Close() failed") } } @@ -386,8 +386,8 @@ func (scw *LegacySplitCloneWorker) findTargets(ctx context.Context) error { } // Initialize healthcheck and add destination shards to it. - scw.healthCheck = discovery.NewHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) - scw.tsc = discovery.NewTabletStatsCache(scw.healthCheck, scw.wr.TopoServer(), scw.cell) + scw.healthCheck = discovery.NewLegacyHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) + scw.tsc = discovery.NewLegacyTabletStatsCache(scw.healthCheck, scw.wr.TopoServer(), scw.cell) for _, si := range scw.destinationShards { watcher := discovery.NewShardReplicationWatcher(ctx, scw.wr.TopoServer(), scw.healthCheck, scw.cell, si.Keyspace(), si.ShardName(), @@ -405,7 +405,7 @@ func (scw *LegacySplitCloneWorker) findTargets(ctx context.Context) error { } masters := scw.tsc.GetHealthyTabletStats(si.Keyspace(), si.ShardName(), topodatapb.TabletType_MASTER) if len(masters) == 0 { - return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v in HealthCheck: empty TabletStats list", si.Keyspace(), si.ShardName()) + return fmt.Errorf("cannot find MASTER tablet for destination shard for %v/%v in LegacyHealthCheck: empty LegacyTabletStats list", si.Keyspace(), si.ShardName()) } master := masters[0] @@ -421,7 +421,7 @@ func (scw *LegacySplitCloneWorker) findTargets(ctx context.Context) error { scw.wr.Logger().Infof("Using tablet %v as destination master for %v/%v", topoproto.TabletAliasString(master.Tablet.Alias), si.Keyspace(), si.ShardName()) } - scw.wr.Logger().Infof("NOTE: The used master of a destination shard might change over the course of the copy e.g. due to a reparent. The HealthCheck module will track and log master changes and any error message will always refer the actually used master address.") + scw.wr.Logger().Infof("NOTE: The used master of a destination shard might change over the course of the copy e.g. due to a reparent. The LegacyHealthCheck module will track and log master changes and any error message will always refer the actually used master address.") // Set up the throttler for each destination shard. for _, si := range scw.destinationShards { diff --git a/go/vt/worker/split_clone.go b/go/vt/worker/split_clone.go index ce042c7d351..f1d479eae1e 100644 --- a/go/vt/worker/split_clone.go +++ b/go/vt/worker/split_clone.go @@ -97,8 +97,8 @@ type SplitCloneWorker struct { // MASTER tablet, b) get the list of healthy RDONLY tablets and c) track the // replication lag of all REPLICA tablets. // It must be closed at the end of the command. - healthCheck discovery.HealthCheck - tsc *discovery.TabletStatsCache + healthCheck discovery.LegacyHealthCheck + tsc *discovery.LegacyTabletStatsCache // populated during WorkerStateFindTargets, read-only after that sourceTablets []*topodatapb.Tablet @@ -420,7 +420,7 @@ func (scw *SplitCloneWorker) Run(ctx context.Context) error { // After Close returned, we can be sure that it won't call our listener // implementation (method StatsUpdate) anymore. if err := scw.healthCheck.Close(); err != nil { - scw.wr.Logger().Errorf2(err, "HealthCheck.Close() failed") + scw.wr.Logger().Errorf2(err, "LegacyHealthCheck.Close() failed") } } @@ -558,9 +558,9 @@ func (scw *SplitCloneWorker) init(ctx context.Context) error { } // Initialize healthcheck and add destination shards to it. - scw.healthCheck = discovery.NewHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) + scw.healthCheck = discovery.NewLegacyHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) scw.tsc = discovery.NewTabletStatsCacheDoNotSetListener(scw.wr.TopoServer(), scw.cell) - // We set sendDownEvents=true because it's required by TabletStatsCache. + // We set sendDownEvents=true because it's required by LegacyTabletStatsCache. scw.healthCheck.SetListener(scw, true /* sendDownEvents */) // Start watchers to get tablets added automatically to healthCheck. @@ -851,7 +851,7 @@ func (scw *SplitCloneWorker) findDestinationMasters(ctx context.Context) error { } masters := scw.tsc.GetHealthyTabletStats(si.Keyspace(), si.ShardName(), topodatapb.TabletType_MASTER) if len(masters) == 0 { - return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "cannot find MASTER tablet for destination shard for %v/%v (in cell: %v) in HealthCheck: empty TabletStats list", si.Keyspace(), si.ShardName(), scw.cell) + return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "cannot find MASTER tablet for destination shard for %v/%v (in cell: %v) in LegacyHealthCheck: empty LegacyTabletStats list", si.Keyspace(), si.ShardName(), scw.cell) } master := masters[0] @@ -861,7 +861,7 @@ func (scw *SplitCloneWorker) findDestinationMasters(ctx context.Context) error { scw.wr.Logger().Infof("Using tablet %v as destination master for %v/%v", topoproto.TabletAliasString(master.Tablet.Alias), si.Keyspace(), si.ShardName()) } - scw.wr.Logger().Infof("NOTE: The used master of a destination shard might change over the course of the copy e.g. due to a reparent. The HealthCheck module will track and log master changes and any error message will always refer the actually used master address.") + scw.wr.Logger().Infof("NOTE: The used master of a destination shard might change over the course of the copy e.g. due to a reparent. The LegacyHealthCheck module will track and log master changes and any error message will always refer the actually used master address.") return nil } @@ -1355,9 +1355,9 @@ func (scw *SplitCloneWorker) createKeyResolver(td *tabletmanagerdatapb.TableDefi // StatsUpdate receives replication lag updates for each destination master // and forwards them to the respective throttler instance. -// It also forwards any update to the TabletStatsCache to keep it up to date. -// It is part of the discovery.HealthCheckStatsListener interface. -func (scw *SplitCloneWorker) StatsUpdate(ts *discovery.TabletStats) { +// It also forwards any update to the LegacyTabletStatsCache to keep it up to date. +// It is part of the discovery.LegacyHealthCheckStatsListener interface. +func (scw *SplitCloneWorker) StatsUpdate(ts *discovery.LegacyTabletStats) { scw.tsc.StatsUpdate(ts) // Ignore unless REPLICA or RDONLY. diff --git a/go/vt/worker/tablet_provider.go b/go/vt/worker/tablet_provider.go index 2eaddf9e8b3..248eba1de84 100644 --- a/go/vt/worker/tablet_provider.go +++ b/go/vt/worker/tablet_provider.go @@ -73,16 +73,16 @@ func (p *singleTabletProvider) description() string { } // shardTabletProvider returns a random healthy RDONLY tablet for a given -// keyspace and shard. It uses the HealthCheck module to retrieve the tablets. +// keyspace and shard. It uses the LegacyHealthCheck module to retrieve the tablets. type shardTabletProvider struct { - tsc *discovery.TabletStatsCache + tsc *discovery.LegacyTabletStatsCache tracker *TabletTracker keyspace string shard string tabletType topodatapb.TabletType } -func newShardTabletProvider(tsc *discovery.TabletStatsCache, tracker *TabletTracker, keyspace, shard string, tabletType topodatapb.TabletType) *shardTabletProvider { +func newShardTabletProvider(tsc *discovery.LegacyTabletStatsCache, tracker *TabletTracker, keyspace, shard string, tabletType topodatapb.TabletType) *shardTabletProvider { return &shardTabletProvider{tsc, tracker, keyspace, shard, tabletType} } diff --git a/go/vt/worker/tablet_tracker.go b/go/vt/worker/tablet_tracker.go index ba85e4ab092..3e2dee1ba61 100644 --- a/go/vt/worker/tablet_tracker.go +++ b/go/vt/worker/tablet_tracker.go @@ -50,7 +50,7 @@ func NewTabletTracker() *TabletTracker { // Track will pick the least used tablet from "stats", increment its usage by 1 // and return it. // "stats" must not be empty. -func (t *TabletTracker) Track(stats []discovery.TabletStats) *topodatapb.Tablet { +func (t *TabletTracker) Track(stats []discovery.LegacyTabletStats) *topodatapb.Tablet { if len(stats) == 0 { panic("stats must not be empty") } diff --git a/go/vt/worker/tablet_tracker_test.go b/go/vt/worker/tablet_tracker_test.go index cf91138cbce..887ce538f6d 100644 --- a/go/vt/worker/tablet_tracker_test.go +++ b/go/vt/worker/tablet_tracker_test.go @@ -27,25 +27,25 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -var ts1 = discovery.TabletStats{ +var ts1 = discovery.LegacyTabletStats{ Tablet: topo.NewTablet(10, "cell", "host1"), Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, } -var ts2 = discovery.TabletStats{ +var ts2 = discovery.LegacyTabletStats{ Tablet: topo.NewTablet(20, "cell", "host1"), Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, } -var allTs = []discovery.TabletStats{ts1, ts2} +var allTs = []discovery.LegacyTabletStats{ts1, ts2} func TestTabletsInUse(t *testing.T) { tt := NewTabletTracker() - tt.Track([]discovery.TabletStats{ts1}) + tt.Track([]discovery.LegacyTabletStats{ts1}) if got, want := tt.TabletsInUse(), "cell-0000000010"; got != want { t.Fatalf("TabletsInUse() = %v, want = %v", got, want) } - tt.Track([]discovery.TabletStats{ts2}) + tt.Track([]discovery.LegacyTabletStats{ts2}) if got, want := tt.TabletsInUse(), "cell-0000000010 cell-0000000020"; got != want { t.Fatalf("TabletsInUse() = %v, want = %v", got, want) } diff --git a/go/vt/worker/topo_utils.go b/go/vt/worker/topo_utils.go index c83f90a2fa1..d63f15705d7 100644 --- a/go/vt/worker/topo_utils.go +++ b/go/vt/worker/topo_utils.go @@ -47,11 +47,11 @@ var ( // Since we don't want to use them all, we require at least // minHealthyRdonlyTablets servers to be healthy. // May block up to -wait_for_healthy_rdonly_tablets_timeout. -func FindHealthyTablet(ctx context.Context, wr *wrangler.Wrangler, tsc *discovery.TabletStatsCache, cell, keyspace, shard string, minHealthyRdonlyTablets int, tabletType topodatapb.TabletType) (*topodatapb.TabletAlias, error) { +func FindHealthyTablet(ctx context.Context, wr *wrangler.Wrangler, tsc *discovery.LegacyTabletStatsCache, cell, keyspace, shard string, minHealthyRdonlyTablets int, tabletType topodatapb.TabletType) (*topodatapb.TabletAlias, error) { if tsc == nil { // No healthcheck instance provided. Create one. - healthCheck := discovery.NewHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) - tsc = discovery.NewTabletStatsCache(healthCheck, wr.TopoServer(), cell) + healthCheck := discovery.NewLegacyHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) + tsc = discovery.NewLegacyTabletStatsCache(healthCheck, wr.TopoServer(), cell) watcher := discovery.NewShardReplicationWatcher(ctx, wr.TopoServer(), healthCheck, cell, keyspace, shard, *healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) defer watcher.Stop() defer healthCheck.Close() @@ -67,7 +67,7 @@ func FindHealthyTablet(ctx context.Context, wr *wrangler.Wrangler, tsc *discover return healthyTablets[index].Tablet.Alias, nil } -func waitForHealthyTablets(ctx context.Context, wr *wrangler.Wrangler, tsc *discovery.TabletStatsCache, cell, keyspace, shard string, minHealthyRdonlyTablets int, timeout time.Duration, tabletType topodatapb.TabletType) ([]discovery.TabletStats, error) { +func waitForHealthyTablets(ctx context.Context, wr *wrangler.Wrangler, tsc *discovery.LegacyTabletStatsCache, cell, keyspace, shard string, minHealthyRdonlyTablets int, timeout time.Duration, tabletType topodatapb.TabletType) ([]discovery.LegacyTabletStats, error) { busywaitCtx, busywaitCancel := context.WithTimeout(ctx, timeout) defer busywaitCancel() @@ -81,7 +81,7 @@ func waitForHealthyTablets(ctx context.Context, wr *wrangler.Wrangler, tsc *disc return nil, vterrors.Wrapf(err, "error waiting for %v tablets for (%v,%v/%v)", tabletType, cell, keyspace, shard) } - var healthyTablets []discovery.TabletStats + var healthyTablets []discovery.LegacyTabletStats for { select { case <-busywaitCtx.Done(): @@ -115,7 +115,7 @@ func waitForHealthyTablets(ctx context.Context, wr *wrangler.Wrangler, tsc *disc // - find a tabletType instance in the keyspace / shard // - mark it as worker // - tag it with our worker process -func FindWorkerTablet(ctx context.Context, wr *wrangler.Wrangler, cleaner *wrangler.Cleaner, tsc *discovery.TabletStatsCache, cell, keyspace, shard string, minHealthyTablets int, tabletType topodatapb.TabletType) (*topodatapb.TabletAlias, error) { +func FindWorkerTablet(ctx context.Context, wr *wrangler.Wrangler, cleaner *wrangler.Cleaner, tsc *discovery.LegacyTabletStatsCache, cell, keyspace, shard string, minHealthyTablets int, tabletType topodatapb.TabletType) (*topodatapb.TabletAlias, error) { tabletAlias, err := FindHealthyTablet(ctx, wr, tsc, cell, keyspace, shard, minHealthyTablets, tabletType) if err != nil { return nil, err diff --git a/go/vt/wrangler/keyspace.go b/go/vt/wrangler/keyspace.go index f4d901c1421..93293c52d31 100644 --- a/go/vt/wrangler/keyspace.go +++ b/go/vt/wrangler/keyspace.go @@ -989,9 +989,9 @@ func (wr *Wrangler) waitForDrainInCell(ctx context.Context, cell, keyspace, shar retryDelay, healthCheckTopologyRefresh, healthcheckRetryDelay, healthCheckTimeout, initialWait time.Duration) error { // Create the healthheck module, with a cache. - hc := discovery.NewHealthCheck(healthcheckRetryDelay, healthCheckTimeout) + hc := discovery.NewLegacyHealthCheck(healthcheckRetryDelay, healthCheckTimeout) defer hc.Close() - tsc := discovery.NewTabletStatsCache(hc, wr.TopoServer(), cell) + tsc := discovery.NewLegacyTabletStatsCache(hc, wr.TopoServer(), cell) // Create a tablet watcher. watcher := discovery.NewShardReplicationWatcher(ctx, wr.TopoServer(), hc, cell, keyspace, shard, healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) @@ -1016,8 +1016,8 @@ func (wr *Wrangler) waitForDrainInCell(ctx context.Context, cell, keyspace, shar startTime := time.Now() for { // map key: tablet uid - drainedHealthyTablets := make(map[uint32]*discovery.TabletStats) - notDrainedHealtyTablets := make(map[uint32]*discovery.TabletStats) + drainedHealthyTablets := make(map[uint32]*discovery.LegacyTabletStats) + notDrainedHealtyTablets := make(map[uint32]*discovery.LegacyTabletStats) healthyTablets := tsc.GetHealthyTabletStats(keyspace, shard, servedType) for _, ts := range healthyTablets { @@ -1060,7 +1060,7 @@ func (wr *Wrangler) waitForDrainInCell(ctx context.Context, cell, keyspace, shar return nil } -func formatTabletStats(ts *discovery.TabletStats) string { +func formatTabletStats(ts *discovery.LegacyTabletStats) string { webURL := "unknown http port" if webPort, ok := ts.Tablet.PortMap["vt"]; ok { webURL = fmt.Sprintf("http://%v:%d/", ts.Tablet.Hostname, webPort) From f543f2112ea1ed49d7706c9e5002b69281fcd523 Mon Sep 17 00:00:00 2001 From: deepthi Date: Sat, 11 Apr 2020 11:57:27 -0700 Subject: [PATCH 02/39] healthcheck: move more code to legacy, and create stubs for new healthcheck Signed-off-by: deepthi --- go/cmd/vtgate/status.go | 12 +- go/cmd/vtgate/vtgate.go | 21 +- go/vt/discovery/healthcheck.go | 1055 +++++++++++++++++ go/vt/discovery/legacy_healthcheck.go | 93 +- go/vt/discovery/legacy_replicationlag.go | 206 ++++ go/vt/discovery/legacy_replicationlag_test.go | 370 ++++++ go/vt/discovery/legacy_tablet_stats_cache.go | 7 +- go/vt/discovery/legacy_topology_watcher.go | 449 +++++++ ...est.go => legacy_topology_watcher_test.go} | 26 +- go/vt/discovery/replicationlag.go | 167 +-- go/vt/discovery/replicationlag_test.go | 179 +-- go/vt/discovery/tablet_picker.go | 4 +- go/vt/discovery/tablet_stats_cache.go | 295 +++++ go/vt/discovery/topology_watcher.go | 123 +- go/vt/discovery/utils.go | 4 +- go/vt/schemamanager/schemaswap/schema_swap.go | 6 +- go/vt/vtctld/realtime_status.go | 6 +- go/vt/vtgate/api.go | 58 +- go/vt/vtgate/discoverygateway.go | 76 +- go/vt/vtgate/discoverygateway_test.go | 24 +- go/vt/vtgate/executor.go | 2 +- go/vt/vtgate/gateway.go | 10 - go/vt/vtgate/scatter_conn.go | 39 +- go/vt/vtgate/tabletgateway.go | 320 +++++ go/vt/vtgate/vtgate.go | 100 +- .../tabletserver/txthrottler/tx_throttler.go | 6 +- go/vt/worker/legacy_split_clone.go | 6 +- go/vt/worker/split_clone.go | 6 +- go/vt/worker/topo_utils.go | 2 +- go/vt/wrangler/keyspace.go | 2 +- 30 files changed, 3109 insertions(+), 565 deletions(-) create mode 100644 go/vt/discovery/healthcheck.go create mode 100644 go/vt/discovery/legacy_replicationlag.go create mode 100644 go/vt/discovery/legacy_replicationlag_test.go create mode 100644 go/vt/discovery/legacy_topology_watcher.go rename go/vt/discovery/{topology_watcher_test.go => legacy_topology_watcher_test.go} (93%) create mode 100644 go/vt/discovery/tablet_stats_cache.go create mode 100644 go/vt/vtgate/tabletgateway.go diff --git a/go/cmd/vtgate/status.go b/go/cmd/vtgate/status.go index 11d5b93c6fb..d714e444b90 100644 --- a/go/cmd/vtgate/status.go +++ b/go/cmd/vtgate/status.go @@ -38,7 +38,13 @@ func addStatusParts(vtg *vtgate.VTGate) { servenv.AddStatusPart("Gateway Status", vtgate.StatusTemplate, func() interface{} { return vtg.GetGatewayCacheStatus() }) - servenv.AddStatusPart("Health Check Cache", discovery.HealthCheckTemplate, func() interface{} { - return legacyHealthCheck.CacheStatus() - }) + if *useLegacyHealthCheck { + servenv.AddStatusPart("Health Check Cache", discovery.HealthCheckTemplate, func() interface{} { + return legacyHealthCheck.CacheStatus() + }) + } else { + servenv.AddStatusPart("Health Check Cache", discovery.HealthCheckTemplate, func() interface{} { + return healthCheck.CacheStatus() + }) + } } diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index bbb69e49971..8cb64d04345 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -37,15 +37,15 @@ import ( ) var ( - cell = flag.String("cell", "test_nj", "cell to use") - retryCount = flag.Int("retry-count", 2, "retry count") - healthCheckRetryDelay = flag.Duration("healthcheck_retry_delay", 2*time.Millisecond, "health check retry delay") - healthCheckTimeout = flag.Duration("healthcheck_timeout", time.Minute, "the health check timeout period") - tabletTypesToWait = flag.String("tablet_types_to_wait", "", "wait till connected for specified tablet types during Gateway initialization") + cell = flag.String("cell", "test_nj", "cell to use") + retryCount = flag.Int("retry-count", 2, "retry count") + tabletTypesToWait = flag.String("tablet_types_to_wait", "", "wait till connected for specified tablet types during Gateway initialization") + useLegacyHealthCheck = flag.Bool("use_legacy_health_check", true, "whether to use the legacy health check") ) var resilientServer *srvtopo.ResilientServer var legacyHealthCheck discovery.LegacyHealthCheck +var healthCheck discovery.HealthCheck func init() { rand.Seed(time.Now().UnixNano()) @@ -75,10 +75,15 @@ func main() { } } - legacyHealthCheck = discovery.NewLegacyHealthCheck(*healthCheckRetryDelay, *healthCheckTimeout) - legacyHealthCheck.RegisterStats() + var vtg *vtgate.VTGate + if *useLegacyHealthCheck { + legacyHealthCheck = discovery.NewLegacyHealthCheck(*vtgate.HealthCheckRetryDelay, *vtgate.HealthCheckTimeout) + legacyHealthCheck.RegisterStats() - vtg := vtgate.LegacyInit(context.Background(), legacyHealthCheck, resilientServer, *cell, *retryCount, tabletTypes) + vtg = vtgate.LegacyInit(context.Background(), legacyHealthCheck, resilientServer, *cell, *retryCount, tabletTypes) + } else { + vtg = vtgate.Init(context.Background(), resilientServer, *cell, *retryCount, tabletTypes) + } servenv.OnRun(func() { // Flags are parsed now. Parse the template using the actual flag value and overwrite the current template. diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go new file mode 100644 index 00000000000..4488bfbb4b4 --- /dev/null +++ b/go/vt/discovery/healthcheck.go @@ -0,0 +1,1055 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package discovery provides a way to discover all tablets e.g. within a +// specific shard and monitor their current health. +// +// Use the HealthCheck object to query for tablets and their health. +// +// For an example how to use the HealthCheck object, see worker/topo_utils.go. +// +// Tablets have to be manually added to the HealthCheck using AddTablet(). +// Alternatively, use a Watcher implementation which will constantly watch +// a source (e.g. the topology) and add and remove tablets as they are +// added or removed from the source. +// For a Watcher example have a look at NewShardReplicationWatcher(). +// +// TabletStatsCache is one implementation, that caches the known tablets +// and the healthy ones per keyspace/shard/tabletType. +// +// Internally, the HealthCheck module is connected to each tablet and has a +// streaming RPC (StreamHealth) open to receive periodic health infos. +package discovery + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "hash/crc32" + "html/template" + "net/http" + "sort" + "strings" + "sync" + "time" + + "vitess.io/vitess/go/flagutil" + "vitess.io/vitess/go/vt/topo" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + "vitess.io/vitess/go/netutil" + "vitess.io/vitess/go/stats" + "vitess.io/vitess/go/sync2" + "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/topotools" + "vitess.io/vitess/go/vt/vttablet/queryservice" + "vitess.io/vitess/go/vt/vttablet/tabletconn" + + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +var ( + hcErrorCounters = stats.NewCountersWithMultiLabels("HealthcheckErrors", "Healthcheck Errors", []string{"Keyspace", "ShardName", "TabletType"}) + hcMasterPromotedCounters = stats.NewCountersWithMultiLabels("HealthcheckMasterPromoted", "Master promoted in keyspace/shard name because of health check errors", []string{"Keyspace", "ShardName"}) + healthcheckOnce sync.Once + TabletURLTemplateString = flag.String("tablet_url_template", "http://{{.GetTabletHostPort}}", "format string describing debug tablet url formatting. See the Go code for getTabletDebugURL() how to customize this.") + tabletURLTemplate *template.Template + + //TODO(deepthi): change these vars back to unexported when discoveryGateway is removed + + // RefreshInterval is the interval at which healthcheck refreshes its list of tablets from topo + RefreshInterval = flag.Duration("tablet_refresh_interval", 1*time.Minute, "tablet refresh interval") + // RefreshKnownTablets tells us whether to process all tablets or only new tablets + RefreshKnownTablets = flag.Bool("tablet_refresh_known_tablets", true, "tablet refresh reloads the tablet address/port map from topo in case it changes") + // TopoReadConcurrency tells us how many topo reads are allowed in parallel + TopoReadConcurrency = flag.Int("topo_read_concurrency", 32, "concurrent topo reads") + // CellsToWatch is the list of cells this healthcheck operates over + CellsToWatch = flag.String("cells_to_watch", "", "comma-separated list of cells for watching tablets") + // AllowedTabletTypes is the list of allowed tablet types. e.g. {MASTER, REPLICA} + AllowedTabletTypes []topodatapb.TabletType + // TabletFilters are the keyspace|shard or keyrange filters to apply to the full set of tablets + TabletFilters flagutil.StringListValue + // KeyspacesToWatch - if provided this specifies which keyspaces should be + // visible to a vtgate. By default the vtgate will allow access to any + // keyspace. + KeyspacesToWatch flagutil.StringListValue +) + +// See the documentation for NewLegacyHealthCheck below for an explanation of these parameters. +const ( + DefaultHealthCheckRetryDelay = 5 * time.Second + DefaultHealthCheckTimeout = 1 * time.Minute + + // DefaultTopoReadConcurrency can be used as default value for the TopoReadConcurrency parameter of a LegacyTopologyWatcher. + DefaultTopoReadConcurrency int = 5 + // DefaultTopologyWatcherRefreshInterval can be used as the default value for + // the refresh interval of a topology watcher. + DefaultTopologyWatcherRefreshInterval = 1 * time.Minute + + // HealthCheckTemplate is the HTML code to display a TabletsCacheStatusList + HealthCheckTemplate = ` + + + + + + + + + + + + + {{range $i, $ts := .}} + + + + + + + + {{end}} +
HealthCheck Tablet Cache
CellKeyspaceShardTabletTypeTabletStats
{{github_com_vitessio_vitess_vtctld_srv_cell $ts.Cell}}{{github_com_vitessio_vitess_vtctld_srv_keyspace $ts.Cell $ts.Target.Keyspace}}{{$ts.Target.Shard}}{{$ts.Target.TabletType}}{{$ts.StatusAsHTML}}
+` +) + +// ParseTabletURLTemplateFromFlag loads or reloads the URL template. +func ParseTabletURLTemplateFromFlag() { + tabletURLTemplate = template.New("") + _, err := tabletURLTemplate.Parse(*TabletURLTemplateString) + if err != nil { + log.Exitf("error parsing template: %v", err) + } +} + +func init() { + // Flags are not parsed at this point and the default value of the flag (just the hostname) will be used. + ParseTabletURLTemplateFromFlag() + flag.Var(&TabletFilters, "tablet_filters", "Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch") + topoproto.TabletTypeListVar(&AllowedTabletTypes, "allowed_tablet_types", "Specifies the tablet types this vtgate is allowed to route queries to") + flag.Var(&KeyspacesToWatch, "keyspaces_to_watch", "Specifies which keyspaces this vtgate should have access to while routing queries or accessing the vschema") +} + +// TabletStats is returned when getting the set of tablets. +type TabletStats struct { + // Key uniquely identifies that serving tablet. It is computed + // from the Tablet's record Hostname and PortMap. If a tablet + // is restarted on different ports, its Key will be different. + // Key is computed using the TabletToMapKey method below. + // key can be used in GetConnection(). + Key string + // Tablet is the tablet object that was sent to HealthCheck.AddTablet. + Tablet *topodatapb.Tablet + // Name is an optional tag (e.g. alternative address) for the + // tablet. It is supposed to represent the tablet as a task, + // not as a process. For instance, it can be a + // cell+keyspace+shard+tabletType+taskIndex value. + Name string + // Target is the current target as returned by the streaming + // StreamHealth RPC. + Target *querypb.Target + // Up describes whether the tablet is added or removed. + Up bool + // Serving describes if the tablet can be serving traffic. + Serving bool + // TabletExternallyReparentedTimestamp is the last timestamp + // that this tablet was either elected the master, or received + // a TabletExternallyReparented event. It is set to 0 if the + // tablet doesn't think it's a master. + TabletExternallyReparentedTimestamp int64 + // Stats is the current health status, as received by the + // StreamHealth RPC (replication lag, ...). + Stats *querypb.RealtimeStats + // LastError is the error we last saw when trying to get the + // tablet's healthcheck. + LastError error + // TODO(deepthi): No member of this struct should be accessed without holding the mutex + // mu sync.Mutex +} + +// String is defined because we want to print a []*TabletStats array nicely. +func (e *TabletStats) String() string { + return fmt.Sprint(*e) +} + +// DeepEqual compares two TabletStats. Since we include protos, we +// need to use proto.Equal on these. +func (e *TabletStats) DeepEqual(f *TabletStats) bool { + return e.Key == f.Key && + proto.Equal(e.Tablet, f.Tablet) && + e.Name == f.Name && + proto.Equal(e.Target, f.Target) && + e.Up == f.Up && + e.Serving == f.Serving && + e.TabletExternallyReparentedTimestamp == f.TabletExternallyReparentedTimestamp && + proto.Equal(e.Stats, f.Stats) && + ((e.LastError == nil && f.LastError == nil) || + (e.LastError != nil && f.LastError != nil && e.LastError.Error() == f.LastError.Error())) +} + +// Copy produces a copy of TabletStats. +func (e *TabletStats) Copy() *TabletStats { + ts := *e + return &ts +} + +// GetTabletHostPort formats a tablet host port address. +func (e TabletStats) GetTabletHostPort() string { + vtPort := e.Tablet.PortMap["vt"] + return netutil.JoinHostPort(e.Tablet.Hostname, vtPort) +} + +// GetHostNameLevel returns the specified hostname level. If the level does not exist it will pick the closest level. +// This seems unused but can be utilized by certain url formatting templates. See getTabletDebugURL for more details. +func (e TabletStats) GetHostNameLevel(level int) string { + chunkedHostname := strings.Split(e.Tablet.Hostname, ".") + + if level < 0 { + return chunkedHostname[0] + } else if level >= len(chunkedHostname) { + return chunkedHostname[len(chunkedHostname)-1] + } else { + return chunkedHostname[level] + } +} + +// getTabletDebugURL formats a debug url to the tablet. +// It uses a format string that can be passed into the app to format +// the debug URL to accommodate different network setups. It applies +// the html/template string defined to a TabletStats object. The +// format string can refer to members and functions of TabletStats +// like a regular html/template string. +// +// For instance given a tablet with hostname:port of host.dc.domain:22 +// could be configured as follows: +// http://{{.GetTabletHostPort}} -> http://host.dc.domain:22 +// https://{{.Tablet.Hostname}} -> https://host.dc.domain +// https://{{.GetHostNameLevel 0}}.bastion.corp -> https://host.bastion.corp +func (e TabletStats) getTabletDebugURL() string { + var buffer bytes.Buffer + tabletURLTemplate.Execute(&buffer, e) + return buffer.String() +} + +// TrivialStatsUpdate returns true iff the old and new TabletStats +// haven't changed enough to warrant re-calling FilterLegacyStatsByReplicationLag. +func (e *TabletStats) TrivialStatsUpdate(n *TabletStats) bool { + // Skip replag filter when replag remains in the low rep lag range, + // which should be the case majority of the time. + lowRepLag := lowReplicationLag.Seconds() + oldRepLag := float64(e.Stats.SecondsBehindMaster) + newRepLag := float64(n.Stats.SecondsBehindMaster) + if oldRepLag <= lowRepLag && newRepLag <= lowRepLag { + return true + } + + // Skip replag filter when replag remains in the high rep lag range, + // and did not change beyond +/- 10%. + // when there is a high rep lag, it takes a long time for it to reduce, + // so it is not necessary to re-calculate every time. + // In that case, we won't save the new record, so we still + // remember the original replication lag. + if oldRepLag > lowRepLag && newRepLag > lowRepLag && newRepLag < oldRepLag*1.1 && newRepLag > oldRepLag*0.9 { + return true + } + + return false +} + +// TabletRecorder is the part of the HealthCheck interface that can +// add or remove tablets. We define it as a sub-interface here so we +// can add filters on tablets if needed. +type TabletRecorder interface { + // AddTablet adds the tablet. + // Name is an alternate name, like an address. + AddTablet(tablet *topodatapb.Tablet, name string) + + // RemoveTablet removes the tablet. + RemoveTablet(tablet *topodatapb.Tablet) + + // ReplaceTablet does an AddTablet and RemoveTablet in one call, effectively replacing the old tablet with the new. + ReplaceTablet(old, new *topodatapb.Tablet, name string) +} + +// HealthCheck defines the interface of health checking module. +// The goal of this object is to maintain a StreamHealth RPC +// to a lot of tablets. Tablets are added / removed by calling the +// AddTablet / RemoveTablet methods (other discovery module objects +// can for instance watch the topology and call these). +// +// Updates to the health of all registered tablet can be watched by +// registering a listener. To get the underlying "TabletConn" object +// which is used for each tablet, use the "GetConnection()" method +// below and pass in the Key string which is also sent to the +// listener in each update (as it is part of TabletStats). +type HealthCheck interface { + TabletRecorder + // RegisterStats registers the connection counts and checksum stats. + // It can only be called on one Healthcheck object per process. + RegisterStats() + // WaitForInitialStatsUpdates waits until all tablets added via + // AddTablet() call were propagated to the listener via corresponding + // StatsUpdate() calls. Note that code path from AddTablet() to + // corresponding StatsUpdate() is asynchronous but not cancelable, thus + // this function is also non-cancelable and can't return error. Also + // note that all AddTablet() calls should happen before calling this + // method. WaitForInitialStatsUpdates won't wait for StatsUpdate() calls + // corresponding to AddTablet() calls made during its execution. + WaitForInitialStatsUpdates() + // GetConnection returns the TabletConn of the given tablet. + GetConnection(key string) queryservice.QueryService + // CacheStatus returns a displayable version of the cache. + CacheStatus() TabletsCacheStatusList + // Close stops the healthcheck. + Close() error + // GetHealthyTabletStats gets the tabletStats by tablet type + GetHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats +} + +type tabletFilterFunc func(tablet *topodatapb.Tablet) bool + +// HealthCheckImpl performs health checking and notifies downstream components about any changes. +// It contains a map of TabletHealth objects, each of which stores the health information for +// a tablet. A checkConn goroutine is spawned for each TabletHealth, which is responsible for +// keeping that TabletHealth up-to-date. This is done through callbacks to updateHealth. +// If checkConn terminates for any reason, it updates TabletHealth.Up as false. If a TabletHealth +// gets removed from the map, its cancelFunc gets called, which ensures that the associated +// checkConn goroutine eventually terminates. +type HealthCheckImpl struct { + // Immutable fields set at construction time. + retryDelay time.Duration + healthCheckTimeout time.Duration + // connsWG keeps track of all launched Go routines that monitor tablet connections. + connsWG sync.WaitGroup + + // mu protects all the following fields. + mu sync.Mutex + + // addrToHealth maps from address to TabletHealth. + addrToHealth map[string]*tabletHealth + + // Wait group that's used to wait until all initial StatsUpdate() calls are made after the AddTablet() calls. + initialUpdatesWG sync.WaitGroup + + // ts is the topo server in use. + ts *topo.Server + cell string + tsc *TabletStatsCache + + tabletFilters []tabletFilterFunc + + topoWatchers []*TopologyWatcher +} + +// HealthCheckConn is a structure that lives within the scope of +// the checkConn goroutine to maintain its internal state. Therefore, +// it does not require synchronization. Changes that are relevant to +// healthcheck are transmitted through calls to HealthCheckImpl.updateHealth. +// TODO(deepthi): replace with goroutine (already using a goroutine to update this) +type healthCheckConn struct { + ctx context.Context + + conn queryservice.QueryService + tabletStats TabletStats + loggedServingState bool + lastResponseTimestamp time.Time // timestamp of the last healthcheck response +} + +// TabletHealth maintains the health status of a tablet. A map of this +// structure is maintained in HealthCheckImpl. +type tabletHealth struct { + // cancelFunc must be called before discarding TabletHealth. + // This will ensure that the associated checkConn goroutine will terminate. + cancelFunc context.CancelFunc + // conn is the connection associated with the tablet. + conn queryservice.QueryService + // latestTabletStats stores the latest health stats of the tablet. + latestTabletStats TabletStats +} + +// NewHealthCheck creates a new HealthCheck object. +// Parameters: +// retryDelay. +// The duration to wait before retrying to connect (e.g. after a failed connection +// attempt). +// healthCheckTimeout. +// The duration for which we consider a health check response to be 'fresh'. If we don't get +// a health check response from a tablet for more than this duration, we consider the tablet +// not healthy. +func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string) HealthCheck { + log.Infof("loading tablets for cells: %v", *CellsToWatch) + var filterFuncs []tabletFilterFunc + + for _, c := range strings.Split(*CellsToWatch, ",") { + if c == "" { + continue + } + if len(TabletFilters) > 0 { + if len(KeyspacesToWatch) > 0 { + log.Exitf("Only one of -keyspaces_to_watch and -tablet_filters may be specified at a time") + } + + fbs, err := NewFilterByShard(c, TabletFilters) + if err != nil { + log.Exitf("Cannot parse tablet_filters parameter: %v", err) + } + filterFuncs = append(filterFuncs, fbs.IsIncluded) + } else if len(KeyspacesToWatch) > 0 { + fbk := NewFilterByKeyspace(c, KeyspacesToWatch) + filterFuncs = append(filterFuncs, fbk.IsIncluded) + } + //ctw := NewCellTabletsWatcher(ctx, topoServer, hc, c, *RefreshInterval, + // *RefreshKnownTablets, *TopoReadConcurrency) + } + + hc := &HealthCheckImpl{ + ts: topoServer, + cell: localCell, + addrToHealth: make(map[string]*tabletHealth), + retryDelay: retryDelay, + healthCheckTimeout: healthCheckTimeout, + tsc: newTabletStatsCache(localCell), + tabletFilters: filterFuncs, + } + + // create a go func per cell - call watchCell to watch topo and update list of tablets + + healthcheckOnce.Do(func() { + http.Handle("/debug/gateway", hc) + }) + + return hc +} + +// TODO(deepthi): implement the watch +//func watchCell(ctx context.Context, topoServer *topo.Server, hc *HealthCheck, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) { +// +//} + +// Open starts healthcheck +func (*HealthCheckImpl) Open() { + // create a local cancelable context, cancel it in Close +} + +// RegisterStats registers the connection counts stats +func (hc *HealthCheckImpl) RegisterStats() { + stats.NewGaugeDurationFunc( + "TopologyWatcherMaxRefreshLag", + "maximum time since the topology watcher refreshed a cell", + hc.topologyWatcherMaxRefreshLag, + ) + + stats.NewGaugeFunc( + "TopologyWatcherChecksum", + "crc32 checksum of the topology watcher state", + hc.topologyWatcherChecksum, + ) + + stats.NewGaugesFuncWithMultiLabels( + "HealthcheckConnections", + "the number of healthcheck connections registered", + []string{"Keyspace", "ShardName", "TabletType"}, + hc.servingConnStats) + + stats.NewGaugeFunc( + "HealthcheckChecksum", + "crc32 checksum of the current healthcheck state", + hc.stateChecksum) +} + +// ServeHTTP is part of the http.Handler interface. It renders the current state of the discovery gateway tablet cache into json. +func (hc *HealthCheckImpl) ServeHTTP(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + status := hc.cacheStatusMap() + b, err := json.MarshalIndent(status, "", " ") + if err != nil { + w.Write([]byte(err.Error())) + return + } + + buf := bytes.NewBuffer(nil) + json.HTMLEscape(buf, b) + w.Write(buf.Bytes()) +} + +// servingConnStats returns the number of serving tablets per keyspace/shard/tablet type. +func (hc *HealthCheckImpl) servingConnStats() map[string]int64 { + res := make(map[string]int64) + hc.mu.Lock() + defer hc.mu.Unlock() + for _, th := range hc.addrToHealth { + if !th.latestTabletStats.Up || !th.latestTabletStats.Serving || th.latestTabletStats.LastError != nil { + continue + } + key := fmt.Sprintf("%s.%s.%s", th.latestTabletStats.Target.Keyspace, th.latestTabletStats.Target.Shard, topoproto.TabletTypeLString(th.latestTabletStats.Target.TabletType)) + res[key]++ + } + return res +} + +// stateChecksum returns a crc32 checksum of the healthcheck state +func (hc *HealthCheckImpl) stateChecksum() int64 { + // CacheStatus is sorted so this should be stable across vtgates + cacheStatus := hc.CacheStatus() + var buf bytes.Buffer + for _, st := range cacheStatus { + fmt.Fprintf(&buf, + "%v%v%v%v\n", + st.Cell, + st.Target.Keyspace, + st.Target.Shard, + st.Target.TabletType.String(), + ) + sort.Sort(st.TabletsStats) + for _, ts := range st.TabletsStats { + fmt.Fprintf(&buf, "%v%v%v\n", ts.Up, ts.Serving, ts.TabletExternallyReparentedTimestamp) + } + } + + return int64(crc32.ChecksumIEEE(buf.Bytes())) +} + +// updateHealth updates the TabletHealth record and updates the tablet stats +func (hc *HealthCheckImpl) updateHealth(ts *TabletStats, conn queryservice.QueryService) { + // directly update hc.tsc here + hc.tsc.UpdateStats(ts, hc.ts) + + hc.mu.Lock() + th, ok := hc.addrToHealth[ts.Key] + if !ok { + // This can happen on delete because the entry is removed first, + // or if HealthCheckImpl has been closed. + hc.mu.Unlock() + return + } + + // TODO(deepthi): do we need to make a copy of the stats? + oldts := th.latestTabletStats + th.latestTabletStats = *ts + th.conn = conn + hc.mu.Unlock() + + // In the case where a tablet changes type (but not for the + // initial message), we want to log it, and maybe advertise it too. + if oldts.Target.TabletType != topodatapb.TabletType_UNKNOWN && oldts.Target.TabletType != ts.Target.TabletType { + // Log and maybe notify + log.Infof("HealthCheckUpdate(Type Change): %v, tablet: %s, target %+v => %+v, reparent time: %v", + oldts.Name, topotools.TabletIdent(oldts.Tablet), topotools.TargetIdent(oldts.Target), topotools.TargetIdent(ts.Target), ts.TabletExternallyReparentedTimestamp) + //TODO(deepthi): directly update hc.tsc here + //if hc.listener != nil && hc.sendDownEvents { + //oldts.Up = false + //hc.listener.StatsUpdate(&oldts) + //} + + // Track how often a tablet gets promoted to master. It is used for + // comparing against the variables in go/vtgate/buffer/variables.go. + if oldts.Target.TabletType != topodatapb.TabletType_MASTER && ts.Target.TabletType == topodatapb.TabletType_MASTER { + hcMasterPromotedCounters.Add([]string{ts.Target.Keyspace, ts.Target.Shard}, 1) + } + } +} + +// finalizeConn closes the health checking connection and sends the final +// notification about the tablet to downstream. To be called only on exit from +// checkConn(). +func (hc *HealthCheckImpl) finalizeConn(hcc *healthCheckConn) { + hcc.tabletStats.Up = false + hcc.setServingState(false, "finalizeConn closing connection") + // Note: checkConn() exits only when hcc.ctx.Done() is closed. Thus it's + // safe to simply get Err() value here and assign to LastError. + hcc.tabletStats.LastError = hcc.ctx.Err() + hc.updateHealth(hcc.tabletStats.Copy(), nil) + if hcc.conn != nil { + // Don't use hcc.ctx because it's already closed. + // Use a separate context, and add a timeout to prevent unbounded waits. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + hcc.conn.Close(ctx) + hcc.conn = nil + } +} + +// checkConn performs health checking on the given tablet. +func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn, name string) { + defer hc.connsWG.Done() + defer hc.finalizeConn(hcc) + + // Initial notification for downstream about the tablet existence. + // do not copy + hc.updateHealth(hcc.tabletStats.Copy(), hcc.conn) + hc.initialUpdatesWG.Done() + + retryDelay := hc.retryDelay + for { + streamCtx, streamCancel := context.WithCancel(hcc.ctx) + + // Setup a watcher that restarts the timer every time an update is received. + // If a timeout occurs for a serving tablet, we make it non-serving and send + // a status update. The stream is also terminated so it can be retried. + // servingStatus feeds into the serving var, which keeps track of the serving + // status transmitted by the tablet. + servingStatus := make(chan bool, 1) + // timedout is accessed atomically because there could be a race + // between the goroutine that sets it and the check for its value + // later. + timedout := sync2.NewAtomicBool(false) + go func() { + for { + select { + case <-servingStatus: + continue + case <-time.After(hc.healthCheckTimeout): + timedout.Set(true) + streamCancel() + return + case <-streamCtx.Done(): + // If the stream is done, stop watching. + return + } + } + }() + + // Read stream health responses. + hcc.stream(streamCtx, hc, func(shr *querypb.StreamHealthResponse) error { + // We received a message. Reset the back-off. + retryDelay = hc.retryDelay + // Don't block on send to avoid deadlocks. + select { + case servingStatus <- shr.Serving: + default: + } + return hcc.processResponse(hc, shr) + }) + + // streamCancel to make sure the watcher goroutine terminates. + streamCancel() + + // If there was a timeout send an error. We do this after stream has returned. + // This will ensure that this update prevails over any previous message that + // stream could have sent. + if timedout.Get() { + hcc.tabletStats.LastError = fmt.Errorf("healthcheck timed out (latest %v)", hcc.lastResponseTimestamp) + hcc.setServingState(false, hcc.tabletStats.LastError.Error()) + hc.updateHealth(hcc.tabletStats.Copy(), hcc.conn) + hcErrorCounters.Add([]string{hcc.tabletStats.Target.Keyspace, hcc.tabletStats.Target.Shard, topoproto.TabletTypeLString(hcc.tabletStats.Target.TabletType)}, 1) + } + + // Streaming RPC failed e.g. because vttablet was restarted or took too long. + // Sleep until the next retry is up or the context is done/canceled. + select { + case <-hcc.ctx.Done(): + return + case <-time.After(retryDelay): + // Exponentially back-off to prevent tight-loop. + retryDelay *= 2 + // Limit the retry delay backoff to the health check timeout + if retryDelay > hc.healthCheckTimeout { + retryDelay = hc.healthCheckTimeout + } + } + } +} + +// setServingState sets the tablet state to the given value. +// +// If the state changes, it logs the change so that failures +// from the health check connection are logged the first time, +// but don't continue to log if the connection stays down. +// +// hcc.mu must be locked before calling this function +func (hcc *healthCheckConn) setServingState(serving bool, reason string) { + if !hcc.loggedServingState || (serving != hcc.tabletStats.Serving) { + // Emit the log from a separate goroutine to avoid holding + // the hcc lock while logging is happening + go log.Infof("HealthCheckUpdate(Serving State): %v, tablet: %v serving => %v for %v/%v (%v) reason: %s", + hcc.tabletStats.Name, + topotools.TabletIdent(hcc.tabletStats.Tablet), + serving, + hcc.tabletStats.Tablet.GetKeyspace(), + hcc.tabletStats.Tablet.GetShard(), + hcc.tabletStats.Target.GetTabletType(), + reason, + ) + hcc.loggedServingState = true + } + + hcc.tabletStats.Serving = serving +} + +// stream streams healthcheck responses to callback. +func (hcc *healthCheckConn) stream(ctx context.Context, hc *HealthCheckImpl, callback func(*querypb.StreamHealthResponse) error) { + if hcc.conn == nil { + conn, err := tabletconn.GetDialer()(hcc.tabletStats.Tablet, grpcclient.FailFast(true)) + if err != nil { + hcc.tabletStats.LastError = err + return + } + hcc.conn = conn + hcc.tabletStats.LastError = nil + } + + if err := hcc.conn.StreamHealth(ctx, callback); err != nil { + log.Warningf("tablet %v healthcheck stream error: %v", hcc.tabletStats.Tablet.Alias, err) + hcc.setServingState(false, err.Error()) + hcc.tabletStats.LastError = err + // Send nil because we intend to close the connection. + hc.updateHealth(hcc.tabletStats.Copy(), nil) + hcc.conn.Close(ctx) + hcc.conn = nil + } +} + +// processResponse reads one health check response, and updates health +func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.StreamHealthResponse) error { + select { + case <-hcc.ctx.Done(): + return hcc.ctx.Err() + default: + } + + // Check for invalid data, better than panicking. + if shr.Target == nil || shr.RealtimeStats == nil { + return fmt.Errorf("health stats is not valid: %v", shr) + } + + // an app-level error from tablet, force serving state. + var healthErr error + serving := shr.Serving + if shr.RealtimeStats.HealthError != "" { + healthErr = fmt.Errorf("vttablet error: %v", shr.RealtimeStats.HealthError) + serving = false + } + + // hcc.TabletStats.Tablet.Alias.Uid may be 0 because the youtube internal mechanism uses a different + // code path to initialize this value. If so, we should skip this check. + if shr.TabletAlias != nil && hcc.tabletStats.Tablet.Alias.Uid != 0 && !proto.Equal(shr.TabletAlias, hcc.tabletStats.Tablet.Alias) { + return fmt.Errorf("health stats mismatch, tablet %+v alias does not match response alias %v", hcc.tabletStats.Tablet, shr.TabletAlias) + } + + // In this case where a new tablet is initialized or a tablet type changes, we want to + // initialize the counter so the rate can be calculated correctly. + if hcc.tabletStats.Target.TabletType != shr.Target.TabletType { + hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) + } + + // Update our record, and notify downstream for tabletType and + // realtimeStats change. + hcc.lastResponseTimestamp = time.Now() + hcc.tabletStats.Target = shr.Target + hcc.tabletStats.TabletExternallyReparentedTimestamp = shr.TabletExternallyReparentedTimestamp + hcc.tabletStats.Stats = shr.RealtimeStats + hcc.tabletStats.LastError = healthErr + reason := "healthCheck update" + if healthErr != nil { + reason = "healthCheck update error: " + healthErr.Error() + } + hcc.setServingState(serving, reason) + + // TODO(deepthi): do we need a copy? updateHealth should be done by HealthCheckConn, not healthCheck + hc.updateHealth(hcc.tabletStats.Copy(), hcc.conn) + return nil +} + +func (hc *HealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { + hc.mu.Lock() + defer hc.mu.Unlock() + + key := TabletToMapKey(tablet) + th, ok := hc.addrToHealth[key] + if !ok { + return + } + // Make sure the key still corresponds to the tablet we want to delete. + // If it doesn't match, we should do nothing. The tablet we were asked to + // delete is already gone, and some other tablet is using the key + // (host:port) that the original tablet used to use, which is fine. + if !topoproto.TabletAliasEqual(tablet.Alias, th.latestTabletStats.Tablet.Alias) { + return + } + hc.deleteConnLocked(key, th) +} + +func (hc *HealthCheckImpl) deleteConnLocked(key string, th *tabletHealth) { + th.latestTabletStats.Up = false + th.cancelFunc() + delete(hc.addrToHealth, key) +} + +// AddTablet adds the tablet, and starts health check. +// It does not block on making connection. +// name is an optional tag for the tablet, e.g. an alternative address. +func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet, name string) { + ctx, cancelFunc := context.WithCancel(context.Background()) + key := TabletToMapKey(tablet) + hcc := &healthCheckConn{ + ctx: ctx, + tabletStats: TabletStats{ + Key: key, + Tablet: tablet, + Name: name, + Target: &querypb.Target{}, + Up: true, + }, + } + hc.mu.Lock() + if hc.addrToHealth == nil { + // already closed. + hc.mu.Unlock() + return + } + if th, ok := hc.addrToHealth[key]; ok { + // Something already exists at this key. + // If it's the same tablet, something is wrong. + if topoproto.TabletAliasEqual(th.latestTabletStats.Tablet.Alias, tablet.Alias) { + hc.mu.Unlock() + log.Warningf("refusing to add duplicate tablet %v for %v: %+v", name, tablet.Alias.Cell, tablet) + return + } + // If it's a different tablet, then we trust this new tablet that claims + // it has taken over the host:port that the old tablet used to be on. + // Remove the old tablet to clear the way. + hc.deleteConnLocked(key, th) + } + hc.addrToHealth[key] = &tabletHealth{ + cancelFunc: cancelFunc, + latestTabletStats: hcc.tabletStats, + } + hc.initialUpdatesWG.Add(1) + hc.connsWG.Add(1) + hc.mu.Unlock() + + // checkConn should take a TabletStats + // should exit when healthcheck context is canceled + go hc.checkConn(hcc, name) +} + +// RemoveTablet removes the tablet, and stops the health check. +// It does not block. +func (hc *HealthCheckImpl) RemoveTablet(tablet *topodatapb.Tablet) { + hc.deleteConn(tablet) +} + +// ReplaceTablet removes the old tablet and adds the new tablet. +func (hc *HealthCheckImpl) ReplaceTablet(old, new *topodatapb.Tablet, name string) { + hc.deleteConn(old) + hc.AddTablet(new, name) +} + +// WaitForInitialStatsUpdates waits until all tablets added via AddTablet() call +// were propagated to downstream via corresponding StatsUpdate() calls. +func (hc *HealthCheckImpl) WaitForInitialStatsUpdates() { + hc.initialUpdatesWG.Wait() +} + +// GetConnection returns the TabletConn of the given tablet. +func (hc *HealthCheckImpl) GetConnection(key string) queryservice.QueryService { + hc.mu.Lock() + defer hc.mu.Unlock() + + th := hc.addrToHealth[key] + if th == nil { + return nil + } + return th.conn +} + +// TabletsCacheStatus is the current tablets for a cell/target. +type TabletsCacheStatus struct { + Cell string + Target *querypb.Target + TabletsStats TabletStatsList +} + +// TabletStatsList is used for sorting. +type TabletStatsList []*TabletStats + +// Len is part of sort.Interface. +func (tsl TabletStatsList) Len() int { + return len(tsl) +} + +// Less is part of sort.Interface +func (tsl TabletStatsList) Less(i, j int) bool { + name1 := tsl[i].Name + if name1 == "" { + name1 = tsl[i].Key + } + name2 := tsl[j].Name + if name2 == "" { + name2 = tsl[j].Key + } + return name1 < name2 +} + +// Swap is part of sort.Interface +func (tsl TabletStatsList) Swap(i, j int) { + tsl[i], tsl[j] = tsl[j], tsl[i] +} + +// StatusAsHTML returns an HTML version of the status. +func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML { + tLinks := make([]string, 0, 1) + if tcs.TabletsStats != nil { + sort.Sort(tcs.TabletsStats) + } + for _, ts := range tcs.TabletsStats { + color := "green" + extra := "" + if ts.LastError != nil { + color = "red" + extra = fmt.Sprintf(" (%v)", ts.LastError) + } else if !ts.Serving { + color = "red" + extra = " (Not Serving)" + } else if !ts.Up { + color = "red" + extra = " (Down)" + } else if ts.Target.TabletType == topodatapb.TabletType_MASTER { + extra = fmt.Sprintf(" (MasterTS: %v)", ts.TabletExternallyReparentedTimestamp) + } else { + extra = fmt.Sprintf(" (RepLag: %v)", ts.Stats.SecondsBehindMaster) + } + name := ts.Name + if name == "" { + name = ts.GetTabletHostPort() + } + tLinks = append(tLinks, fmt.Sprintf(`%v%v`, ts.getTabletDebugURL(), color, name, extra)) + } + return template.HTML(strings.Join(tLinks, "
")) +} + +// TabletsCacheStatusList is used for sorting. +type TabletsCacheStatusList []*TabletsCacheStatus + +// Len is part of sort.Interface. +func (tcsl TabletsCacheStatusList) Len() int { + return len(tcsl) +} + +// Less is part of sort.Interface +func (tcsl TabletsCacheStatusList) Less(i, j int) bool { + return tcsl[i].Cell+"."+tcsl[i].Target.Keyspace+"."+tcsl[i].Target.Shard+"."+string(tcsl[i].Target.TabletType) < + tcsl[j].Cell+"."+tcsl[j].Target.Keyspace+"."+tcsl[j].Target.Shard+"."+string(tcsl[j].Target.TabletType) +} + +// Swap is part of sort.Interface +func (tcsl TabletsCacheStatusList) Swap(i, j int) { + tcsl[i], tcsl[j] = tcsl[j], tcsl[i] +} + +// CacheStatus returns a displayable version of the cache. +func (hc *HealthCheckImpl) CacheStatus() TabletsCacheStatusList { + tcsMap := hc.cacheStatusMap() + tcsl := make(TabletsCacheStatusList, 0, len(tcsMap)) + for _, tcs := range tcsMap { + tcsl = append(tcsl, tcs) + } + sort.Sort(tcsl) + return tcsl +} + +func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { + tcsMap := make(map[string]*TabletsCacheStatus) + hc.mu.Lock() + defer hc.mu.Unlock() + for _, th := range hc.addrToHealth { + key := fmt.Sprintf("%v.%v.%v.%v", th.latestTabletStats.Tablet.Alias.Cell, th.latestTabletStats.Target.Keyspace, th.latestTabletStats.Target.Shard, th.latestTabletStats.Target.TabletType.String()) + var tcs *TabletsCacheStatus + var ok bool + if tcs, ok = tcsMap[key]; !ok { + tcs = &TabletsCacheStatus{ + Cell: th.latestTabletStats.Tablet.Alias.Cell, + Target: th.latestTabletStats.Target, + } + tcsMap[key] = tcs + } + stats := th.latestTabletStats + tcs.TabletsStats = append(tcs.TabletsStats, &stats) + } + return tcsMap +} + +// Close stops the healthcheck. +func (hc *HealthCheckImpl) Close() error { + hc.mu.Lock() + for _, th := range hc.addrToHealth { + th.cancelFunc() + } + hc.addrToHealth = nil + // Release the lock early or a pending checkHealthCheckTimeout + // cannot get a read lock on it. + hc.mu.Unlock() + + // Wait for the checkHealthCheckTimeout Go routine and each Go + // routine per tablet. + hc.connsWG.Wait() + + return nil +} + +// topologyWatcherMaxRefreshLag returns the maximum lag since the watched +// cells were refreshed from the topo server +func (hc *HealthCheckImpl) topologyWatcherMaxRefreshLag() time.Duration { + var lag time.Duration + for _, tw := range hc.topoWatchers { + cellLag := tw.RefreshLag() + if cellLag > lag { + lag = cellLag + } + } + return lag +} + +// topologyWatcherChecksum returns a checksum of the topology watcher state +func (hc *HealthCheckImpl) topologyWatcherChecksum() int64 { + var checksum int64 + for _, tw := range hc.topoWatchers { + checksum = checksum ^ int64(tw.TopoChecksum()) + } + return checksum +} + +// GetHealthyTabletStats returns only the healthy targets. +// The returned array is owned by the caller. +// For TabletType_MASTER, this will only return at most one entry, +// the most recent tablet of type master. +func (hc *HealthCheckImpl) GetHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats { + e := hc.tsc.getEntry(keyspace, shard, tabletType) + if e == nil { + return nil + } + + e.mu.RLock() + defer e.mu.RUnlock() + result := make([]TabletStats, len(e.healthy)) + for i, ts := range e.healthy { + result[i] = *ts + } + return result +} diff --git a/go/vt/discovery/legacy_healthcheck.go b/go/vt/discovery/legacy_healthcheck.go index 91317ecb931..fec2d63eb2b 100644 --- a/go/vt/discovery/legacy_healthcheck.go +++ b/go/vt/discovery/legacy_healthcheck.go @@ -25,7 +25,7 @@ limitations under the License. // Alternatively, use a Watcher implementation which will constantly watch // a source (e.g. the topology) and add and remove tablets as they are // added or removed from the source. -// For a Watcher example have a look at NewShardReplicationWatcher(). +// For a Watcher example have a look at NewLegacyShardReplicationWatcher(). // // Each LegacyHealthCheck has a LegacyHealthCheckStatsListener that will receive // notification of when tablets go up and down. @@ -39,7 +39,6 @@ package discovery import ( "bytes" "encoding/json" - "flag" "fmt" "hash/crc32" "html/template" @@ -66,76 +65,11 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -var ( - hcErrorCounters = stats.NewCountersWithMultiLabels("HealthcheckErrors", "Healthcheck Errors", []string{"Keyspace", "ShardName", "TabletType"}) - hcMasterPromotedCounters = stats.NewCountersWithMultiLabels("HealthcheckMasterPromoted", "Master promoted in keyspace/shard name because of health check errors", []string{"Keyspace", "ShardName"}) - healthcheckOnce sync.Once - - // TabletURLTemplateString is a flag to generate URLs for the tablets that vtgate discovers. - TabletURLTemplateString = flag.String("tablet_url_template", "http://{{.GetTabletHostPort}}", "format string describing debug tablet url formatting. See the Go code for getTabletDebugURL() how to customize this.") - tabletURLTemplate *template.Template -) - -// See the documentation for NewLegacyHealthCheck below for an explanation of these parameters. -const ( - DefaultHealthCheckRetryDelay = 5 * time.Second - DefaultHealthCheckTimeout = 1 * time.Minute - - // DefaultTopoReadConcurrency can be used as default value for the topoReadConcurrency parameter of a TopologyWatcher. - DefaultTopoReadConcurrency int = 5 - // DefaultTopologyWatcherRefreshInterval can be used as the default value for - // the refresh interval of a topology watcher. - DefaultTopologyWatcherRefreshInterval = 1 * time.Minute - - // HealthCheckTemplate is the HTML code to display a TabletsCacheStatusList - HealthCheckTemplate = ` - - - - - - - - - - - - - {{range $i, $ts := .}} - - - - - - - - {{end}} -
HealthCheck Tablet Cache
CellKeyspaceShardTabletTypeTabletStats
{{github_com_vitessio_vitess_vtctld_srv_cell $ts.Cell}}{{github_com_vitessio_vitess_vtctld_srv_keyspace $ts.Cell $ts.Target.Keyspace}}{{$ts.Target.Shard}}{{$ts.Target.TabletType}}{{$ts.StatusAsHTML}}
-` -) - func init() { // Flags are not parsed at this point and the default value of the flag (just the hostname) will be used. ParseTabletURLTemplateFromFlag() } -// ParseTabletURLTemplateFromFlag loads or reloads the URL template. -func ParseTabletURLTemplateFromFlag() { - tabletURLTemplate = template.New("") - _, err := tabletURLTemplate.Parse(*TabletURLTemplateString) - if err != nil { - log.Exitf("error parsing template: %v", err) - } -} - // LegacyHealthCheckStatsListener is the listener to receive health check stats update. type LegacyHealthCheckStatsListener interface { // StatsUpdate is called when: @@ -256,6 +190,31 @@ func (e LegacyTabletStats) getTabletDebugURL() string { return buffer.String() } +// TrivialStatsUpdate returns true iff the old and new LegacyTabletStats +// haven't changed enough to warrant re-calling FilterLegacyStatsByReplicationLag. +func (e *LegacyTabletStats) TrivialStatsUpdate(n *LegacyTabletStats) bool { + // Skip replag filter when replag remains in the low rep lag range, + // which should be the case majority of the time. + lowRepLag := lowReplicationLag.Seconds() + oldRepLag := float64(e.Stats.SecondsBehindMaster) + newRepLag := float64(n.Stats.SecondsBehindMaster) + if oldRepLag <= lowRepLag && newRepLag <= lowRepLag { + return true + } + + // Skip replag filter when replag remains in the high rep lag range, + // and did not change beyond +/- 10%. + // when there is a high rep lag, it takes a long time for it to reduce, + // so it is not necessary to re-calculate every time. + // In that case, we won't save the new record, so we still + // remember the original replication lag. + if oldRepLag > lowRepLag && newRepLag > lowRepLag && newRepLag < oldRepLag*1.1 && newRepLag > oldRepLag*0.9 { + return true + } + + return false +} + // LegacyHealthCheck defines the interface of health checking module. // The goal of this object is to maintain a StreamHealth RPC // to a lot of tablets. Tablets are added / removed by calling the diff --git a/go/vt/discovery/legacy_replicationlag.go b/go/vt/discovery/legacy_replicationlag.go new file mode 100644 index 00000000000..e7eedbbb2de --- /dev/null +++ b/go/vt/discovery/legacy_replicationlag.go @@ -0,0 +1,206 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "flag" + "fmt" + "sort" +) + +var ( + legacyReplicationLagAlgorithm = flag.Bool("legacy_replication_lag_algorithm", true, "use the legacy algorithm when selecting the vttablets for serving") +) + +// LegacyIsReplicationLagHigh verifies that the given LegacyTabletStats refers to a tablet with high +// replication lag, i.e. higher than the configured discovery_low_replication_lag flag. +func LegacyIsReplicationLagHigh(tabletStats *LegacyTabletStats) bool { + return float64(tabletStats.Stats.SecondsBehindMaster) > lowReplicationLag.Seconds() +} + +// LegacyIsReplicationLagVeryHigh verifies that the given LegacyTabletStats refers to a tablet with very high +// replication lag, i.e. higher than the configured discovery_high_replication_lag_minimum_serving flag. +func LegacyIsReplicationLagVeryHigh(tabletStats *LegacyTabletStats) bool { + return float64(tabletStats.Stats.SecondsBehindMaster) > highReplicationLagMinServing.Seconds() +} + +// FilterLegacyStatsByReplicationLag filters the list of LegacyTabletStats by LegacyTabletStats.Stats.SecondsBehindMaster. +// Note that LegacyTabletStats that is non-serving or has error is ignored. +// +// The simplified logic: +// - Return tablets that have lag <= lowReplicationLag. +// - Make sure we return at least minNumTablets tablets, if there are enough one with lag <= highReplicationLagMinServing. +// For example, with the default of 30s / 2h / 2, this means: +// - lags of (5s, 10s, 15s, 120s) return the first three +// - lags of (30m, 35m, 40m, 45m) return the first two +// - lags of (2h, 3h, 4h, 5h) return the first one +// +// The legacy algorithm (default for now): +// - Return the list if there is 0 or 1 tablet. +// - Return the list if all tablets have <=30s lag. +// - Filter by replication lag: for each tablet, if the mean value without it is more than 0.7 of the mean value across all tablets, it is valid. +// - Make sure we return at least minNumTablets tablets (if there are enough one with only low replication lag). +// - If one tablet is removed, run above steps again in case there are two tablets with high replication lag. (It should cover most cases.) +// For example, lags of (5s, 10s, 15s, 120s) return the first three; +// lags of (30m, 35m, 40m, 45m) return all. +// +// One thing to know about this code: vttablet also has a couple flags that impact the logic here: +// * unhealthy_threshold: if replication lag is higher than this, a tablet will be reported as unhealthy. +// The default for this is 2h, same as the discovery_high_replication_lag_minimum_serving here. +// * degraded_threshold: this is only used by vttablet for display. It should match +// discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. +func FilterLegacyStatsByReplicationLag(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { + if !*legacyReplicationLagAlgorithm { + return filterLegacyStatsByLag(tabletStatsList) + } + + res := filterLegacyStatsByLagWithLegacyAlgorithm(tabletStatsList) + // run the filter again if exactly one tablet is removed, + // and we have spare tablets. + if len(res) > *minNumTablets && len(res) == len(tabletStatsList)-1 { + res = filterLegacyStatsByLagWithLegacyAlgorithm(res) + } + return res +} + +func filterLegacyStatsByLag(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { + list := make([]legacyTabletLagSnapshot, 0, len(tabletStatsList)) + // filter non-serving tablets and those with very high replication lag + for _, ts := range tabletStatsList { + if !ts.Serving || ts.LastError != nil || ts.Stats == nil || LegacyIsReplicationLagVeryHigh(ts) { + continue + } + // Pull the current replication lag for a stable sort later. + list = append(list, legacyTabletLagSnapshot{ + ts: ts, + replag: ts.Stats.SecondsBehindMaster}) + } + + // Sort by replication lag. + sort.Sort(byReplag(list)) + + // Pick those with low replication lag, but at least minNumTablets tablets regardless. + res := make([]*LegacyTabletStats, 0, len(list)) + for i := 0; i < len(list); i++ { + if !LegacyIsReplicationLagHigh(list[i].ts) || i < *minNumTablets { + res = append(res, list[i].ts) + } + } + return res +} + +func filterLegacyStatsByLagWithLegacyAlgorithm(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { + list := make([]*LegacyTabletStats, 0, len(tabletStatsList)) + // filter non-serving tablets + for _, ts := range tabletStatsList { + if !ts.Serving || ts.LastError != nil || ts.Stats == nil { + continue + } + list = append(list, ts) + } + if len(list) <= 1 { + return list + } + // if all have low replication lag (<=30s), return all tablets. + allLowLag := true + for _, ts := range list { + if LegacyIsReplicationLagHigh(ts) { + allLowLag = false + break + } + } + if allLowLag { + return list + } + // filter those affecting "mean" lag significantly + // calculate mean for all tablets + res := make([]*LegacyTabletStats, 0, len(list)) + m, _ := mean(list, -1) + for i, ts := range list { + // calculate mean by excluding ith tablet + mi, _ := mean(list, i) + if float64(mi) > float64(m)*0.7 { + res = append(res, ts) + } + } + if len(res) >= *minNumTablets { + return res + } + // return at least minNumTablets tablets to avoid over loading, + // if there is enough tablets with replication lag < highReplicationLagMinServing. + // Pull the current replication lag for a stable sort. + snapshots := make([]legacyTabletLagSnapshot, 0, len(list)) + for _, ts := range list { + if !LegacyIsReplicationLagVeryHigh(ts) { + snapshots = append(snapshots, legacyTabletLagSnapshot{ + ts: ts, + replag: ts.Stats.SecondsBehindMaster}) + } + } + if len(snapshots) == 0 { + // We get here if all tablets are over the high + // replication lag threshold, and their lag is + // different enough that the 70% mean computation up + // there didn't find them all in a group. For + // instance, if *minNumTablets = 2, and we have two + // tablets with lag of 3h and 30h. In that case, we + // just use them all. + for _, ts := range list { + snapshots = append(snapshots, legacyTabletLagSnapshot{ + ts: ts, + replag: ts.Stats.SecondsBehindMaster}) + } + } + + // Sort by replication lag. + sort.Sort(byReplag(snapshots)) + + // Pick the first minNumTablets tablets. + res = make([]*LegacyTabletStats, 0, *minNumTablets) + for i := 0; i < min(*minNumTablets, len(snapshots)); i++ { + res = append(res, snapshots[i].ts) + } + return res +} + +type legacyTabletLagSnapshot struct { + ts *LegacyTabletStats + replag uint32 +} +type byReplag []legacyTabletLagSnapshot + +func (a byReplag) Len() int { return len(a) } +func (a byReplag) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byReplag) Less(i, j int) bool { return a[i].replag < a[j].replag } + +// mean calculates the mean value over the given list, +// while excluding the item with the specified index. +func mean(tabletStatsList []*LegacyTabletStats, idxExclude int) (uint64, error) { + var sum uint64 + var count uint64 + for i, ts := range tabletStatsList { + if i == idxExclude { + continue + } + sum = sum + uint64(ts.Stats.SecondsBehindMaster) + count++ + } + if count == 0 { + return 0, fmt.Errorf("empty list") + } + return sum / count, nil +} diff --git a/go/vt/discovery/legacy_replicationlag_test.go b/go/vt/discovery/legacy_replicationlag_test.go new file mode 100644 index 00000000000..e6026ea40a0 --- /dev/null +++ b/go/vt/discovery/legacy_replicationlag_test.go @@ -0,0 +1,370 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "fmt" + "testing" + + querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/topo" +) + +// testSetLegacyReplicationLagAlgorithm is a test helper function, if this is used by a production code path, something is wrong. +func testSetLegacyReplicationLagAlgorithm(newLegacy bool) { + *legacyReplicationLagAlgorithm = newLegacy +} + +func TestFilterLegacyStatsByReplicationLagUnhealthy(t *testing.T) { + // 1 healthy serving tablet, 1 not healhty + ts1 := &LegacyTabletStats{ + Tablet: topo.NewTablet(1, "cell", "host1"), + Serving: true, + Stats: &querypb.RealtimeStats{}, + } + ts2 := &LegacyTabletStats{ + Tablet: topo.NewTablet(2, "cell", "host2"), + Serving: false, + Stats: &querypb.RealtimeStats{}, + } + got := FilterLegacyStatsByReplicationLag([]*LegacyTabletStats{ts1, ts2}) + if len(got) != 1 { + t.Errorf("len(FilterLegacyStatsByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}])) = %v, want 1", len(got)) + } + if len(got) > 0 && !got[0].DeepEqual(ts1) { + t.Errorf("FilterLegacyStatsByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}]) = %+v, want %+v", got[0], ts1) + } +} + +func TestFilterLegacyStatsByReplicationLag(t *testing.T) { + // Use simplified logic + testSetLegacyReplicationLagAlgorithm(false) + + cases := []struct { + description string + input []uint32 + output []uint32 + }{ + { + "0 tablet", + []uint32{}, + []uint32{}, + }, + { + "lags of (1s) - return all items with low lag.", + []uint32{1}, + []uint32{1}, + }, + { + "lags of (1s, 1s, 1s, 30s) - return all items with low lag.", + []uint32{1, 1, 1, 30}, + []uint32{1, 1, 1, 30}, + }, + { + "lags of (1s, 1s, 1s, 40m, 40m, 40m) - return all items with low lag.", + []uint32{1, 1, 1, 40 * 60, 40 * 60, 40 * 60}, + []uint32{1, 1, 1}, + }, + { + "lags of (1s, 40m, 40m, 40m) - return at least 2 items if they don't have very high lag.", + []uint32{1, 40 * 60, 40 * 60, 40 * 60}, + []uint32{1, 40 * 60}, + }, + { + "lags of (30m, 35m, 40m, 45m) - return at least 2 items if they don't have very high lag.", + []uint32{30 * 60, 35 * 60, 40 * 60, 45 * 60}, + []uint32{30 * 60, 35 * 60}, + }, + { + "lags of (2h, 3h, 4h, 5h) - return <2 items if the others have very high lag.", + []uint32{2 * 60 * 60, 3 * 60 * 60, 4 * 60 * 60, 5 * 60 * 60}, + []uint32{2 * 60 * 60}, + }, + { + "lags of (3h, 30h) - return nothing if all have very high lag.", + []uint32{3 * 60 * 60, 30 * 60 * 60}, + []uint32{}, + }, + } + + for _, tc := range cases { + lts := make([]*LegacyTabletStats, len(tc.input)) + for i, lag := range tc.input { + lts[i] = &LegacyTabletStats{ + Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, + } + } + got := FilterLegacyStatsByReplicationLag(lts) + if len(got) != len(tc.output) { + t.Errorf("FilterLegacyStatsByReplicationLag(%v) failed: got output:\n%v\nExpected: %v", tc.description, got, tc.output) + continue + } + for i, elag := range tc.output { + if got[i].Stats.SecondsBehindMaster != elag { + t.Errorf("FilterLegacyStatsByReplicationLag(%v) failed: got output:\n%v\nExpected value index %v to be %v", tc.description, got, i, elag) + } + } + } + + // Reset to the default + testSetLegacyReplicationLagAlgorithm(true) +} + +func TestFilterLegacyStatysByReplicationLagWithLegacyAlgorithm(t *testing.T) { + // Use legacy algorithm by default for now + + cases := []struct { + description string + input []uint32 + output []uint32 + }{ + { + "0 tablet", + []uint32{}, + []uint32{}, + }, + { + "1 serving tablet", + []uint32{1}, + []uint32{1}, + }, + { + "lags of (1s, 1s, 1s, 30s)", + []uint32{1, 1, 1, 30}, + []uint32{1, 1, 1, 30}, + }, + { + "lags of (30m, 35m, 40m, 45m)", + []uint32{30 * 60, 35 * 60, 40 * 60, 45 * 60}, + []uint32{30 * 60, 35 * 60, 40 * 60, 45 * 60}, + }, + { + "lags of (1s, 1s, 1m, 40m, 40m) - not run filter the second time as first run removed two items.", + []uint32{1, 1, 60, 40 * 60, 40 * 60}, + []uint32{1, 1, 60}, + }, + { + "lags of (1s, 1s, 10m, 40m) - run filter twice to remove two items", + []uint32{1, 1, 10 * 60, 40 * 60}, + []uint32{1, 1}, + }, + { + "lags of (1m, 100m) - return at least 2 items to avoid overloading if the 2nd one is not delayed too much.", + []uint32{1 * 60, 100 * 60}, + []uint32{1 * 60, 100 * 60}, + }, + { + "lags of (1m, 3h) - return 1 if the 2nd one is delayed too much.", + []uint32{1 * 60, 3 * 60 * 60}, + []uint32{1 * 60}, + }, + { + "lags of (3h) - return 1 as they're all delayed too much.", + []uint32{3 * 60 * 60}, + []uint32{3 * 60 * 60}, + }, + { + "lags of (3h, 4h) - return 2 as they're all delayed too much, but still in a good group.", + []uint32{3 * 60 * 60, 4 * 60 * 60}, + []uint32{3 * 60 * 60, 4 * 60 * 60}, + }, + { + "lags of (3h, 3h, 4h) - return 3 as they're all delayed too much, but still in a good group.", + []uint32{3 * 60 * 60, 3 * 60 * 60, 4 * 60 * 60}, + []uint32{3 * 60 * 60, 3 * 60 * 60, 4 * 60 * 60}, + }, + { + "lags of (3h, 15h, 18h) - return 3 as they're all delayed too much, but still in a good group." + + "(different test case than above to show how absurb the good group logic is)", + []uint32{3 * 60 * 60, 15 * 60 * 60, 18 * 60 * 60}, + []uint32{3 * 60 * 60, 15 * 60 * 60, 18 * 60 * 60}, + }, + { + "lags of (3h, 12h, 18h) - return 2 as they're all delayed too much, but 18h is now considered an outlier." + + "(different test case than above to show how absurb the good group logic is)", + []uint32{3 * 60 * 60, 12 * 60 * 60, 18 * 60 * 60}, + []uint32{3 * 60 * 60, 12 * 60 * 60}, + }, + { + "lags of (3h, 30h) - return 2 as they're all delayed too much." + + "(different test case that before, as both tablet stats are" + + "widely different, not within 70% of eachother)", + []uint32{3 * 60 * 60, 30 * 60 * 60}, + []uint32{3 * 60 * 60, 30 * 60 * 60}, + }, + } + + for _, tc := range cases { + lts := make([]*LegacyTabletStats, len(tc.input)) + for i, lag := range tc.input { + lts[i] = &LegacyTabletStats{ + Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, + } + } + got := FilterLegacyStatsByReplicationLag(lts) + if len(got) != len(tc.output) { + t.Errorf("FilterLegacyStatsByReplicationLag(%v) failed: got output:\n%v\nExpected: %v", tc.description, got, tc.output) + continue + } + for i, elag := range tc.output { + if got[i].Stats.SecondsBehindMaster != elag { + t.Errorf("FilterLegacyStatsByReplicationLag(%v) failed: got output:\n%v\nExpected value index %v to be %v", tc.description, got, i, elag) + } + } + } +} + +func TestFilterLegacyStatsByReplicationLagThreeTabletMin(t *testing.T) { + // Use at least 3 tablets if possible + testSetMinNumTablets(3) + // lags of (1s, 1s, 10m, 11m) - returns at least32 items where the slightly delayed ones that are returned are the 10m and 11m ones. + ts1 := &LegacyTabletStats{ + Tablet: topo.NewTablet(1, "cell", "host1"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, + } + ts2 := &LegacyTabletStats{ + Tablet: topo.NewTablet(2, "cell", "host2"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, + } + ts3 := &LegacyTabletStats{ + Tablet: topo.NewTablet(3, "cell", "host3"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, + } + ts4 := &LegacyTabletStats{ + Tablet: topo.NewTablet(4, "cell", "host4"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, + } + got := FilterLegacyStatsByReplicationLag([]*LegacyTabletStats{ts1, ts2, ts3, ts4}) + if len(got) != 3 || !got[0].DeepEqual(ts1) || !got[1].DeepEqual(ts2) || !got[2].DeepEqual(ts3) { + t.Errorf("FilterLegacyStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) + } + // lags of (11m, 10m, 1s, 1s) - reordered tablets returns the same 3 items where the slightly delayed one that is returned is the 10m and 11m ones. + ts1 = &LegacyTabletStats{ + Tablet: topo.NewTablet(1, "cell", "host1"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, + } + ts2 = &LegacyTabletStats{ + Tablet: topo.NewTablet(2, "cell", "host2"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, + } + ts3 = &LegacyTabletStats{ + Tablet: topo.NewTablet(3, "cell", "host3"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, + } + ts4 = &LegacyTabletStats{ + Tablet: topo.NewTablet(4, "cell", "host4"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, + } + got = FilterLegacyStatsByReplicationLag([]*LegacyTabletStats{ts1, ts2, ts3, ts4}) + if len(got) != 3 || !got[0].DeepEqual(ts3) || !got[1].DeepEqual(ts4) || !got[2].DeepEqual(ts2) { + t.Errorf("FilterLegacyStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) + } + // Reset to the default + testSetMinNumTablets(2) +} + +func TestFilterByReplicationLagOneTabletMin(t *testing.T) { + // Use at least 1 tablets if possible + testSetMinNumTablets(1) + // lags of (1s, 100m) - return only healthy tablet if that is all that is available. + ts1 := &LegacyTabletStats{ + Tablet: topo.NewTablet(1, "cell", "host1"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, + } + ts2 := &LegacyTabletStats{ + Tablet: topo.NewTablet(2, "cell", "host2"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, + } + got := FilterLegacyStatsByReplicationLag([]*LegacyTabletStats{ts1, ts2}) + if len(got) != 1 || !got[0].DeepEqual(ts1) { + t.Errorf("FilterLegacyStatsByReplicationLag([1s, 100m]) = %+v, want [1s]", got) + } + // lags of (1m, 100m) - return only healthy tablet if that is all that is healthy enough. + ts1 = &LegacyTabletStats{ + Tablet: topo.NewTablet(1, "cell", "host1"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60}, + } + ts2 = &LegacyTabletStats{ + Tablet: topo.NewTablet(2, "cell", "host2"), + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, + } + got = FilterLegacyStatsByReplicationLag([]*LegacyTabletStats{ts1, ts2}) + if len(got) != 1 || !got[0].DeepEqual(ts1) { + t.Errorf("FilterLegacyStatsByReplicationLag([1m, 100m]) = %+v, want [1m]", got) + } + // Reset to the default + testSetMinNumTablets(2) +} + +func TestTrivialLegacyStatsUpdate(t *testing.T) { + // Note the healthy threshold is set to 30s. + cases := []struct { + o uint32 + n uint32 + expected bool + }{ + // both are under 30s + {o: 0, n: 1, expected: true}, + {o: 15, n: 20, expected: true}, + + // one is under 30s, the other isn't + {o: 2, n: 40, expected: false}, + {o: 40, n: 10, expected: false}, + + // both are over 30s, but close enough + {o: 100, n: 100, expected: true}, + {o: 100, n: 105, expected: true}, + {o: 105, n: 100, expected: true}, + + // both are over 30s, but too far + {o: 100, n: 120, expected: false}, + {o: 120, n: 100, expected: false}, + } + + for _, c := range cases { + o := &LegacyTabletStats{ + Stats: &querypb.RealtimeStats{ + SecondsBehindMaster: c.o, + }, + } + n := &LegacyTabletStats{ + Stats: &querypb.RealtimeStats{ + SecondsBehindMaster: c.n, + }, + } + got := o.TrivialStatsUpdate(n) + if got != c.expected { + t.Errorf("TrivialStatsUpdate(%v, %v) = %v, expected %v", c.o, c.n, got, c.expected) + } + } +} diff --git a/go/vt/discovery/legacy_tablet_stats_cache.go b/go/vt/discovery/legacy_tablet_stats_cache.go index dc8d981cedb..2c90efba682 100644 --- a/go/vt/discovery/legacy_tablet_stats_cache.go +++ b/go/vt/discovery/legacy_tablet_stats_cache.go @@ -31,7 +31,7 @@ import ( // LegacyTabletStatsCache is a LegacyHealthCheckStatsListener that keeps both the // current list of available LegacyTabletStats, and a serving list: // - for master tablets, only the current master is kept. -// - for non-master tablets, we filter the list using FilterByReplicationLag. +// - for non-master tablets, we filter the list using FilterLegacyStatsByReplicationLag. // It keeps entries for all tablets in the cell(s) it's configured to serve for, // and for the master independently of which cell it's in. // Note the healthy tablet computation is done when we receive a tablet @@ -221,7 +221,8 @@ func (tc *LegacyTabletStatsCache) StatsUpdate(ts *LegacyTabletStats) { if ts.Up { // We have an existing entry, and a new entry. // Remember if they are both good (most common case). - trivialNonMasterUpdate = existing.LastError == nil && existing.Serving && ts.LastError == nil && ts.Serving && ts.Target.TabletType != topodatapb.TabletType_MASTER && TrivialStatsUpdate(existing, ts) + trivialNonMasterUpdate = existing.LastError == nil && existing.Serving && ts.LastError == nil && + ts.Serving && ts.Target.TabletType != topodatapb.TabletType_MASTER && existing.TrivialStatsUpdate(ts) // We already have the entry, update the // values if necessary. (will update both @@ -263,7 +264,7 @@ func (tc *LegacyTabletStatsCache) StatsUpdate(ts *LegacyTabletStats) { for _, s := range e.all { allArray = append(allArray, s) } - e.healthy = FilterByReplicationLag(allArray) + e.healthy = FilterLegacyStatsByReplicationLag(allArray) } } diff --git a/go/vt/discovery/legacy_topology_watcher.go b/go/vt/discovery/legacy_topology_watcher.go new file mode 100644 index 00000000000..b2b407987e3 --- /dev/null +++ b/go/vt/discovery/legacy_topology_watcher.go @@ -0,0 +1,449 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "bytes" + "fmt" + "hash/crc32" + "sort" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + "vitess.io/vitess/go/trace" + + "vitess.io/vitess/go/vt/key" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/topoproto" + + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +// NewLegacyCellTabletsWatcher returns a LegacyTopologyWatcher that monitors all +// the tablets in a cell, and starts refreshing. +func NewLegacyCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *LegacyTopologyWatcher { + return NewLegacyTopologyWatcher(ctx, topoServer, tr, cell, refreshInterval, refreshKnownTablets, topoReadConcurrency, func(tw *LegacyTopologyWatcher) ([]*topodatapb.TabletAlias, error) { + return tw.topoServer.GetTabletsByCell(ctx, tw.cell) + }) +} + +// NewLegacyShardReplicationWatcher returns a LegacyTopologyWatcher that +// monitors the tablets in a cell/keyspace/shard, and starts refreshing. +func NewLegacyShardReplicationWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) *LegacyTopologyWatcher { + return NewLegacyTopologyWatcher(ctx, topoServer, tr, cell, refreshInterval, true /* RefreshKnownTablets */, topoReadConcurrency, func(tw *LegacyTopologyWatcher) ([]*topodatapb.TabletAlias, error) { + sri, err := tw.topoServer.GetShardReplication(ctx, tw.cell, keyspace, shard) + switch { + case err == nil: + // we handle this case after this switch block + case topo.IsErrType(err, topo.NoNode): + // this is not an error + return nil, nil + default: + return nil, err + } + + result := make([]*topodatapb.TabletAlias, len(sri.Nodes)) + for i, node := range sri.Nodes { + result[i] = node.TabletAlias + } + return result, nil + }) +} + +// LegacyTopologyWatcher polls tablet from a configurable set of tablets +// periodically. When tablets are added / removed, it calls +// the TabletRecorder AddTablet / RemoveTablet interface appropriately. +type LegacyTopologyWatcher struct { + // set at construction time + topoServer *topo.Server + tr TabletRecorder + cell string + refreshInterval time.Duration + refreshKnownTablets bool + getTablets func(tw *LegacyTopologyWatcher) ([]*topodatapb.TabletAlias, error) + sem chan int + ctx context.Context + cancelFunc context.CancelFunc + // wg keeps track of all launched Go routines. + wg sync.WaitGroup + + // mu protects all variables below + mu sync.Mutex + // tablets contains a map of alias -> tabletInfo for all known tablets + tablets map[string]*tabletInfo + // topoChecksum stores a crc32 of the tablets map and is exported as a metric + topoChecksum uint32 + // lastRefresh records the timestamp of the last topo refresh + lastRefresh time.Time + // firstLoadDone is true when first load of the topology data is done. + firstLoadDone bool + // firstLoadChan is closed when the initial loading of topology data is done. + firstLoadChan chan struct{} +} + +// NewLegacyTopologyWatcher returns a LegacyTopologyWatcher that monitors all +// the tablets in a cell, and starts refreshing. +func NewLegacyTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *LegacyTopologyWatcher) ([]*topodatapb.TabletAlias, error)) *LegacyTopologyWatcher { + tw := &LegacyTopologyWatcher{ + topoServer: topoServer, + tr: tr, + cell: cell, + refreshInterval: refreshInterval, + refreshKnownTablets: refreshKnownTablets, + getTablets: getTablets, + sem: make(chan int, topoReadConcurrency), + tablets: make(map[string]*tabletInfo), + } + tw.firstLoadChan = make(chan struct{}) + + // We want the span from the context, but not the cancelation that comes with it + spanContext := trace.CopySpan(context.Background(), ctx) + tw.ctx, tw.cancelFunc = context.WithCancel(spanContext) + tw.wg.Add(1) + go tw.watch() + return tw +} + +// watch polls all tablets and notifies TabletRecorder by adding/removing tablets. +func (tw *LegacyTopologyWatcher) watch() { + defer tw.wg.Done() + ticker := time.NewTicker(tw.refreshInterval) + defer ticker.Stop() + for { + tw.loadTablets() + select { + case <-tw.ctx.Done(): + return + case <-ticker.C: + } + } +} + +// loadTablets reads all tablets from topology, and updates TabletRecorder. +func (tw *LegacyTopologyWatcher) loadTablets() { + var wg sync.WaitGroup + newTablets := make(map[string]*tabletInfo) + replacedTablets := make(map[string]*tabletInfo) + + tabletAliases, err := tw.getTablets(tw) + topologyWatcherOperations.Add(topologyWatcherOpListTablets, 1) + if err != nil { + topologyWatcherErrors.Add(topologyWatcherOpListTablets, 1) + select { + case <-tw.ctx.Done(): + return + default: + } + log.Errorf("cannot get tablets for cell: %v: %v", tw.cell, err) + return + } + + // Accumulate a list of all known alias strings to use later + // when sorting + tabletAliasStrs := make([]string, 0, len(tabletAliases)) + + tw.mu.Lock() + for _, tAlias := range tabletAliases { + aliasStr := topoproto.TabletAliasString(tAlias) + tabletAliasStrs = append(tabletAliasStrs, aliasStr) + + if !tw.refreshKnownTablets { + if val, ok := tw.tablets[aliasStr]; ok { + newTablets[aliasStr] = val + continue + } + } + + wg.Add(1) + go func(alias *topodatapb.TabletAlias) { + defer wg.Done() + tw.sem <- 1 // Wait for active queue to drain. + tablet, err := tw.topoServer.GetTablet(tw.ctx, alias) + topologyWatcherOperations.Add(topologyWatcherOpGetTablet, 1) + <-tw.sem // Done; enable next request to run + if err != nil { + topologyWatcherErrors.Add(topologyWatcherOpGetTablet, 1) + select { + case <-tw.ctx.Done(): + return + default: + } + log.Errorf("cannot get tablet for alias %v: %v", alias, err) + return + } + tw.mu.Lock() + aliasStr := topoproto.TabletAliasString(alias) + newTablets[aliasStr] = &tabletInfo{ + alias: aliasStr, + key: TabletToMapKey(tablet.Tablet), + tablet: tablet.Tablet, + } + tw.mu.Unlock() + }(tAlias) + } + + tw.mu.Unlock() + wg.Wait() + tw.mu.Lock() + + for alias, newVal := range newTablets { + if val, ok := tw.tablets[alias]; !ok { + // Check if there's a tablet with the same address key but a + // different alias. If so, replace it and keep track of the + // replaced alias to make sure it isn't removed later. + found := false + for _, otherVal := range tw.tablets { + if newVal.key == otherVal.key { + found = true + tw.tr.ReplaceTablet(otherVal.tablet, newVal.tablet, alias) + topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) + replacedTablets[otherVal.alias] = newVal + } + } + if !found { + tw.tr.AddTablet(newVal.tablet, alias) + topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1) + } + + } else if val.key != newVal.key { + // Handle the case where the same tablet alias is now reporting + // a different address key. + replacedTablets[alias] = newVal + tw.tr.ReplaceTablet(val.tablet, newVal.tablet, alias) + topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) + } + } + + for _, val := range tw.tablets { + if _, ok := newTablets[val.alias]; !ok { + if _, ok2 := replacedTablets[val.alias]; !ok2 { + tw.tr.RemoveTablet(val.tablet) + topologyWatcherOperations.Add(topologyWatcherOpRemoveTablet, 1) + } + } + } + tw.tablets = newTablets + if !tw.firstLoadDone { + tw.firstLoadDone = true + close(tw.firstLoadChan) + } + + // iterate through the tablets in a stable order and compute a + // checksum of the tablet map + sort.Strings(tabletAliasStrs) + var buf bytes.Buffer + for _, alias := range tabletAliasStrs { + tabletInfo, ok := tw.tablets[alias] + if ok { + buf.WriteString(alias) + buf.WriteString(tabletInfo.key) + } + } + tw.topoChecksum = crc32.ChecksumIEEE(buf.Bytes()) + tw.lastRefresh = time.Now() + + tw.mu.Unlock() +} + +// WaitForInitialTopology waits until the watcher reads all of the topology data +// for the first time and transfers the information to TabletRecorder via its +// AddTablet() method. +func (tw *LegacyTopologyWatcher) WaitForInitialTopology() error { + select { + case <-tw.ctx.Done(): + return tw.ctx.Err() + case <-tw.firstLoadChan: + return nil + } +} + +// Stop stops the watcher. It does not clean up the tablets added to TabletRecorder. +func (tw *LegacyTopologyWatcher) Stop() { + tw.cancelFunc() + // wait for watch goroutine to finish. + tw.wg.Wait() +} + +// RefreshLag returns the time since the last refresh +func (tw *LegacyTopologyWatcher) RefreshLag() time.Duration { + tw.mu.Lock() + defer tw.mu.Unlock() + + return time.Since(tw.lastRefresh) +} + +// TopoChecksum returns the checksum of the current state of the topo +func (tw *LegacyTopologyWatcher) TopoChecksum() uint32 { + tw.mu.Lock() + defer tw.mu.Unlock() + + return tw.topoChecksum +} + +// LegacyFilterByShard is a TabletRecorder filter that filters tablets by +// keyspace/shard. +type LegacyFilterByShard struct { + // tr is the underlying TabletRecorder to forward requests too + tr TabletRecorder + + // filters is a map of keyspace to filters for shards + filters map[string][]*filterShard +} + +// NewLegacyFilterByShard creates a new LegacyFilterByShard on top of an existing +// TabletRecorder. Each filter is a keyspace|shard entry, where shard +// can either be a shard name, or a keyrange. All tablets that match +// at least one keyspace|shard tuple will be forwarded to the +// underlying TabletRecorder. +func NewLegacyFilterByShard(tr TabletRecorder, filters []string) (*LegacyFilterByShard, error) { + m := make(map[string][]*filterShard) + for _, filter := range filters { + parts := strings.Split(filter, "|") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid LegacyFilterByShard parameter: %v", filter) + } + + keyspace := parts[0] + shard := parts[1] + + // extract keyrange if it's a range + canonical, kr, err := topo.ValidateShardName(shard) + if err != nil { + return nil, fmt.Errorf("error parsing shard name %v: %v", shard, err) + } + + // check for duplicates + for _, c := range m[keyspace] { + if c.shard == canonical { + return nil, fmt.Errorf("duplicate %v/%v entry", keyspace, shard) + } + } + + m[keyspace] = append(m[keyspace], &filterShard{ + keyspace: keyspace, + shard: canonical, + keyRange: kr, + }) + } + + return &LegacyFilterByShard{ + tr: tr, + filters: m, + }, nil +} + +// AddTablet is part of the TabletRecorder interface. +func (fbs *LegacyFilterByShard) AddTablet(tablet *topodatapb.Tablet, name string) { + if fbs.isIncluded(tablet) { + fbs.tr.AddTablet(tablet, name) + } +} + +// RemoveTablet is part of the TabletRecorder interface. +func (fbs *LegacyFilterByShard) RemoveTablet(tablet *topodatapb.Tablet) { + if fbs.isIncluded(tablet) { + fbs.tr.RemoveTablet(tablet) + } +} + +// ReplaceTablet is part of the TabletRecorder interface. +func (fbs *LegacyFilterByShard) ReplaceTablet(old, new *topodatapb.Tablet, name string) { + if fbs.isIncluded(old) && fbs.isIncluded(new) { + fbs.tr.ReplaceTablet(old, new, name) + } +} + +// isIncluded returns true iff the tablet's keyspace and shard should be +// forwarded to the underlying TabletRecorder. +func (fbs *LegacyFilterByShard) isIncluded(tablet *topodatapb.Tablet) bool { + canonical, kr, err := topo.ValidateShardName(tablet.Shard) + if err != nil { + log.Errorf("Error parsing shard name %v, will ignore tablet: %v", tablet.Shard, err) + return false + } + + for _, c := range fbs.filters[tablet.Keyspace] { + if canonical == c.shard { + // Exact match (probably a non-sharded keyspace). + return true + } + if kr != nil && c.keyRange != nil && key.KeyRangeIncludes(c.keyRange, kr) { + // Our filter's KeyRange includes the provided KeyRange + return true + } + } + return false +} + +// LegacyFilterByKeyspace is a TabletRecorder filter that filters tablets by +// keyspace +type LegacyFilterByKeyspace struct { + tr TabletRecorder + + keyspaces map[string]bool +} + +// NewLegacyFilterByKeyspace creates a new LegacyFilterByKeyspace on top of an existing +// TabletRecorder. Each filter is a keyspace entry. All tablets that match +// a keyspace will be forwarded to the underlying TabletRecorder. +func NewLegacyFilterByKeyspace(tr TabletRecorder, selectedKeyspaces []string) *LegacyFilterByKeyspace { + m := make(map[string]bool) + for _, keyspace := range selectedKeyspaces { + m[keyspace] = true + } + + return &LegacyFilterByKeyspace{ + tr: tr, + keyspaces: m, + } +} + +// AddTablet is part of the TabletRecorder interface. +func (fbk *LegacyFilterByKeyspace) AddTablet(tablet *topodatapb.Tablet, name string) { + if fbk.isIncluded(tablet) { + fbk.tr.AddTablet(tablet, name) + } +} + +// RemoveTablet is part of the TabletRecorder interface. +func (fbk *LegacyFilterByKeyspace) RemoveTablet(tablet *topodatapb.Tablet) { + if fbk.isIncluded(tablet) { + fbk.tr.RemoveTablet(tablet) + } +} + +// ReplaceTablet is part of the TabletRecorder interface. +func (fbk *LegacyFilterByKeyspace) ReplaceTablet(old *topodatapb.Tablet, new *topodatapb.Tablet, name string) { + if old.Keyspace != new.Keyspace { + log.Errorf("Error replacing old tablet in %v with new tablet in %v", old.Keyspace, new.Keyspace) + return + } + + if fbk.isIncluded(new) { + fbk.tr.ReplaceTablet(old, new, name) + } +} + +// isIncluded returns true if the tablet's keyspace should be +// forwarded to the underlying TabletRecorder. +func (fbk *LegacyFilterByKeyspace) isIncluded(tablet *topodatapb.Tablet) bool { + _, exist := fbk.keyspaces[tablet.Keyspace] + return exist +} diff --git a/go/vt/discovery/topology_watcher_test.go b/go/vt/discovery/legacy_topology_watcher_test.go similarity index 93% rename from go/vt/discovery/topology_watcher_test.go rename to go/vt/discovery/legacy_topology_watcher_test.go index 02c664fa15b..8884e2fbdfb 100644 --- a/go/vt/discovery/topology_watcher_test.go +++ b/go/vt/discovery/legacy_topology_watcher_test.go @@ -29,7 +29,7 @@ import ( "vitess.io/vitess/go/vt/topo/memorytopo" ) -func checkOpCounts(t *testing.T, tw *TopologyWatcher, prevCounts, deltas map[string]int64) map[string]int64 { +func checkOpCounts(t *testing.T, tw *LegacyTopologyWatcher, prevCounts, deltas map[string]int64) map[string]int64 { t.Helper() newCounts := topologyWatcherOperations.Counts() for key, prevVal := range prevCounts { @@ -49,7 +49,7 @@ func checkOpCounts(t *testing.T, tw *TopologyWatcher, prevCounts, deltas map[str return newCounts } -func checkChecksum(t *testing.T, tw *TopologyWatcher, want uint32) { +func checkChecksum(t *testing.T, tw *LegacyTopologyWatcher, want uint32) { t.Helper() got := tw.TopoChecksum() if want != got { @@ -75,11 +75,11 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { logger := logutil.NewMemoryLogger() topologyWatcherOperations.ZeroAll() counts := topologyWatcherOperations.Counts() - var tw *TopologyWatcher + var tw *LegacyTopologyWatcher if cellTablets { - tw = NewCellTabletsWatcher(context.Background(), ts, fhc, "aa", 10*time.Minute, refreshKnownTablets, 5) + tw = NewLegacyCellTabletsWatcher(context.Background(), ts, fhc, "aa", 10*time.Minute, refreshKnownTablets, 5) } else { - tw = NewShardReplicationWatcher(context.Background(), ts, fhc, "aa", "keyspace", "shard", 10*time.Minute, 5) + tw = NewLegacyShardReplicationWatcher(context.Background(), ts, fhc, "aa", "keyspace", "shard", 10*time.Minute, 5) } // Wait for the initial topology load to finish. Otherwise we @@ -136,7 +136,7 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { } tw.loadTablets() - // If refreshKnownTablets is disabled, only the new tablet is read + // If RefreshKnownTablets is disabled, only the new tablet is read // from the topo if refreshKnownTablets { counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "AddTablet": 1}) @@ -152,7 +152,7 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { t.Errorf("fhc.GetAllTablets() = %+v; want %+v", allTablets, tablet2) } - // Load the tablets again to show that when refreshKnownTablets is disabled, + // Load the tablets again to show that when RefreshKnownTablets is disabled, // only the list is read from the topo and the checksum doesn't change tw.loadTablets() if refreshKnownTablets { @@ -165,7 +165,7 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { // same tablet, different port, should update (previous // one should go away, new one be added) // - // if refreshKnownTablets is disabled, this case is *not* + // if RefreshKnownTablets is disabled, this case is *not* // detected and the tablet remains in the topo using the // old key origTablet := proto.Clone(tablet).(*topodatapb.Tablet) @@ -207,7 +207,7 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { // trigger a ReplaceTablet in loadTablets because the uid does not // match. // - // This case *is* detected even if refreshKnownTablets is false + // This case *is* detected even if RefreshKnownTablets is false // because the delete tablet / create tablet sequence causes the // list of tablets to change and therefore the change is detected. if err := ts.DeleteTablet(context.Background(), tablet2.Alias); err != nil { @@ -395,9 +395,9 @@ func TestFilterByShard(t *testing.T) { } for _, tc := range testcases { - fbs, err := NewFilterByShard(nil, tc.filters) + fbs, err := NewLegacyFilterByShard(nil, tc.filters) if err != nil { - t.Errorf("cannot create FilterByShard for filters %v: %v", tc.filters, err) + t.Errorf("cannot create LegacyFilterByShard for filters %v: %v", tc.filters, err) } tablet := &topodatapb.Tablet{ @@ -433,9 +433,9 @@ var ( func TestFilterByKeyspace(t *testing.T) { hc := NewFakeHealthCheck() - tr := NewFilterByKeyspace(hc, testKeyspacesToWatch) + tr := NewLegacyFilterByKeyspace(hc, testKeyspacesToWatch) ts := memorytopo.NewServer(testCell) - tw := NewCellTabletsWatcher(context.Background(), ts, tr, testCell, 10*time.Minute, true, 5) + tw := NewLegacyCellTabletsWatcher(context.Background(), ts, tr, testCell, 10*time.Minute, true, 5) for _, test := range testFilterByKeyspace { // Add a new tablet to the topology. diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index c2bbe9569a1..8c5eae415fd 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -18,32 +18,30 @@ package discovery import ( "flag" - "fmt" "sort" "time" ) var ( // lowReplicationLag defines the duration that replication lag is low enough that the VTTablet is considered healthy. - lowReplicationLag = flag.Duration("discovery_low_replication_lag", 30*time.Second, "the replication lag that is considered low enough to be healthy") - highReplicationLagMinServing = flag.Duration("discovery_high_replication_lag_minimum_serving", 2*time.Hour, "the replication lag that is considered too high when selecting the minimum num vttablets for serving") - minNumTablets = flag.Int("min_number_serving_vttablets", 2, "the minimum number of vttablets that will be continue to be used even with low replication lag") - legacyReplicationLagAlgorithm = flag.Bool("legacy_replication_lag_algorithm", true, "use the legacy algorithm when selecting the vttablets for serving") + lowReplicationLag = flag.Duration("discovery_low_replication_lag", 30*time.Second, "the replication lag that is considered low enough to be healthy") + highReplicationLagMinServing = flag.Duration("discovery_high_replication_lag_minimum_serving", 2*time.Hour, "the replication lag that is considered too high when selecting the minimum num vttablets for serving") + minNumTablets = flag.Int("min_number_serving_vttablets", 2, "the minimum number of vttablets that will be continue to be used even with low replication lag") ) // IsReplicationLagHigh verifies that the given LegacyTabletStats refers to a tablet with high // replication lag, i.e. higher than the configured discovery_low_replication_lag flag. -func IsReplicationLagHigh(tabletStats *LegacyTabletStats) bool { +func IsReplicationLagHigh(tabletStats *TabletStats) bool { return float64(tabletStats.Stats.SecondsBehindMaster) > lowReplicationLag.Seconds() } // IsReplicationLagVeryHigh verifies that the given LegacyTabletStats refers to a tablet with very high // replication lag, i.e. higher than the configured discovery_high_replication_lag_minimum_serving flag. -func IsReplicationLagVeryHigh(tabletStats *LegacyTabletStats) bool { +func IsReplicationLagVeryHigh(tabletStats *TabletStats) bool { return float64(tabletStats.Stats.SecondsBehindMaster) > highReplicationLagMinServing.Seconds() } -// FilterByReplicationLag filters the list of LegacyTabletStats by LegacyTabletStats.Stats.SecondsBehindMaster. +// FilterStatsByReplicationLag filters the list of LegacyTabletStats by LegacyTabletStats.Stats.SecondsBehindMaster. // Note that LegacyTabletStats that is non-serving or has error is ignored. // // The simplified logic: @@ -68,21 +66,11 @@ func IsReplicationLagVeryHigh(tabletStats *LegacyTabletStats) bool { // The default for this is 2h, same as the discovery_high_replication_lag_minimum_serving here. // * degraded_threshold: this is only used by vttablet for display. It should match // discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. -func FilterByReplicationLag(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { - if !*legacyReplicationLagAlgorithm { - return filterByLag(tabletStatsList) - } - - res := filterByLagWithLegacyAlgorithm(tabletStatsList) - // run the filter again if exactly one tablet is removed, - // and we have spare tablets. - if len(res) > *minNumTablets && len(res) == len(tabletStatsList)-1 { - res = filterByLagWithLegacyAlgorithm(res) - } - return res +func FilterStatsByReplicationLag(tabletStatsList []*TabletStats) []*TabletStats { + return filterStatsByLag(tabletStatsList) } -func filterByLag(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { +func filterStatsByLag(tabletStatsList []*TabletStats) []*TabletStats { list := make([]tabletLagSnapshot, 0, len(tabletStatsList)) // filter non-serving tablets and those with very high replication lag for _, ts := range tabletStatsList { @@ -96,10 +84,10 @@ func filterByLag(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { } // Sort by replication lag. - sort.Sort(byReplag(list)) + sort.Sort(tabletLagSnapshotList(list)) // Pick those with low replication lag, but at least minNumTablets tablets regardless. - res := make([]*LegacyTabletStats, 0, len(list)) + res := make([]*TabletStats, 0, len(list)) for i := 0; i < len(list); i++ { if !IsReplicationLagHigh(list[i].ts) || i < *minNumTablets { res = append(res, list[i].ts) @@ -108,79 +96,15 @@ func filterByLag(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { return res } -func filterByLagWithLegacyAlgorithm(tabletStatsList []*LegacyTabletStats) []*LegacyTabletStats { - list := make([]*LegacyTabletStats, 0, len(tabletStatsList)) - // filter non-serving tablets - for _, ts := range tabletStatsList { - if !ts.Serving || ts.LastError != nil || ts.Stats == nil { - continue - } - list = append(list, ts) - } - if len(list) <= 1 { - return list - } - // if all have low replication lag (<=30s), return all tablets. - allLowLag := true - for _, ts := range list { - if IsReplicationLagHigh(ts) { - allLowLag = false - break - } - } - if allLowLag { - return list - } - // filter those affecting "mean" lag significantly - // calculate mean for all tablets - res := make([]*LegacyTabletStats, 0, len(list)) - m, _ := mean(list, -1) - for i, ts := range list { - // calculate mean by excluding ith tablet - mi, _ := mean(list, i) - if float64(mi) > float64(m)*0.7 { - res = append(res, ts) - } - } - if len(res) >= *minNumTablets { - return res - } - // return at least minNumTablets tablets to avoid over loading, - // if there is enough tablets with replication lag < highReplicationLagMinServing. - // Pull the current replication lag for a stable sort. - snapshots := make([]tabletLagSnapshot, 0, len(list)) - for _, ts := range list { - if !IsReplicationLagVeryHigh(ts) { - snapshots = append(snapshots, tabletLagSnapshot{ - ts: ts, - replag: ts.Stats.SecondsBehindMaster}) - } - } - if len(snapshots) == 0 { - // We get here if all tablets are over the high - // replication lag threshold, and their lag is - // different enough that the 70% mean computation up - // there didn't find them all in a group. For - // instance, if *minNumTablets = 2, and we have two - // tablets with lag of 3h and 30h. In that case, we - // just use them all. - for _, ts := range list { - snapshots = append(snapshots, tabletLagSnapshot{ - ts: ts, - replag: ts.Stats.SecondsBehindMaster}) - } - } - - // Sort by replication lag. - sort.Sort(byReplag(snapshots)) - - // Pick the first minNumTablets tablets. - res = make([]*LegacyTabletStats, 0, *minNumTablets) - for i := 0; i < min(*minNumTablets, len(snapshots)); i++ { - res = append(res, snapshots[i].ts) - } - return res +type tabletLagSnapshot struct { + ts *TabletStats + replag uint32 } +type tabletLagSnapshotList []tabletLagSnapshot + +func (a tabletLagSnapshotList) Len() int { return len(a) } +func (a tabletLagSnapshotList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a tabletLagSnapshotList) Less(i, j int) bool { return a[i].replag < a[j].replag } func min(a, b int) int { if a > b { @@ -188,56 +112,3 @@ func min(a, b int) int { } return a } - -type tabletLagSnapshot struct { - ts *LegacyTabletStats - replag uint32 -} -type byReplag []tabletLagSnapshot - -func (a byReplag) Len() int { return len(a) } -func (a byReplag) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a byReplag) Less(i, j int) bool { return a[i].replag < a[j].replag } - -// mean calculates the mean value over the given list, -// while excluding the item with the specified index. -func mean(tabletStatsList []*LegacyTabletStats, idxExclude int) (uint64, error) { - var sum uint64 - var count uint64 - for i, ts := range tabletStatsList { - if i == idxExclude { - continue - } - sum = sum + uint64(ts.Stats.SecondsBehindMaster) - count++ - } - if count == 0 { - return 0, fmt.Errorf("empty list") - } - return sum / count, nil -} - -// TrivialStatsUpdate returns true iff the old and new LegacyTabletStats -// haven't changed enough to warrant re-calling FilterByReplicationLag. -func TrivialStatsUpdate(o, n *LegacyTabletStats) bool { - // Skip replag filter when replag remains in the low rep lag range, - // which should be the case majority of the time. - lowRepLag := lowReplicationLag.Seconds() - oldRepLag := float64(o.Stats.SecondsBehindMaster) - newRepLag := float64(n.Stats.SecondsBehindMaster) - if oldRepLag <= lowRepLag && newRepLag <= lowRepLag { - return true - } - - // Skip replag filter when replag remains in the high rep lag range, - // and did not change beyond +/- 10%. - // when there is a high rep lag, it takes a long time for it to reduce, - // so it is not necessary to re-calculate every time. - // In that case, we won't save the new record, so we still - // remember the original replication lag. - if oldRepLag > lowRepLag && newRepLag > lowRepLag && newRepLag < oldRepLag*1.1 && newRepLag > oldRepLag*0.9 { - return true - } - - return false -} diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index f7958b7d3af..40b001aaa1b 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -29,29 +29,24 @@ func testSetMinNumTablets(newMin int) { *minNumTablets = newMin } -// testSetLegacyReplicationLagAlgorithm is a test helper function, if this is used by a production code path, something is wrong. -func testSetLegacyReplicationLagAlgorithm(newLegacy bool) { - *legacyReplicationLagAlgorithm = newLegacy -} - func TestFilterByReplicationLagUnhealthy(t *testing.T) { // 1 healthy serving tablet, 1 not healhty - ts1 := &LegacyTabletStats{ + ts1 := &TabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{}, } - ts2 := &LegacyTabletStats{ + ts2 := &TabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: false, Stats: &querypb.RealtimeStats{}, } - got := FilterByReplicationLag([]*LegacyTabletStats{ts1, ts2}) + got := FilterStatsByReplicationLag([]*TabletStats{ts1, ts2}) if len(got) != 1 { - t.Errorf("len(FilterByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}])) = %v, want 1", len(got)) + t.Errorf("len(FilterStatsByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}])) = %v, want 1", len(got)) } if len(got) > 0 && !got[0].DeepEqual(ts1) { - t.Errorf("FilterByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}]) = %+v, want %+v", got[0], ts1) + t.Errorf("FilterStatsByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}]) = %+v, want %+v", got[0], ts1) } } @@ -107,22 +102,22 @@ func TestFilterByReplicationLag(t *testing.T) { } for _, tc := range cases { - lts := make([]*LegacyTabletStats, len(tc.input)) + lts := make([]*TabletStats, len(tc.input)) for i, lag := range tc.input { - lts[i] = &LegacyTabletStats{ + lts[i] = &TabletStats{ Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, } } - got := FilterByReplicationLag(lts) + got := FilterStatsByReplicationLag(lts) if len(got) != len(tc.output) { - t.Errorf("FilterByReplicationLag(%v) failed: got output:\n%v\nExpected: %v", tc.description, got, tc.output) + t.Errorf("FilterStatsByReplicationLag(%v) failed: got output:\n%v\nExpected: %v", tc.description, got, tc.output) continue } for i, elag := range tc.output { if got[i].Stats.SecondsBehindMaster != elag { - t.Errorf("FilterByReplicationLag(%v) failed: got output:\n%v\nExpected value index %v to be %v", tc.description, got, i, elag) + t.Errorf("FilterStatsByReplicationLag(%v) failed: got output:\n%v\nExpected value index %v to be %v", tc.description, got, i, elag) } } } @@ -131,201 +126,95 @@ func TestFilterByReplicationLag(t *testing.T) { testSetLegacyReplicationLagAlgorithm(true) } -func TestFilterByReplicationLagWithLegacyAlgorithm(t *testing.T) { - // Use legacy algorithm by default for now - - cases := []struct { - description string - input []uint32 - output []uint32 - }{ - { - "0 tablet", - []uint32{}, - []uint32{}, - }, - { - "1 serving tablet", - []uint32{1}, - []uint32{1}, - }, - { - "lags of (1s, 1s, 1s, 30s)", - []uint32{1, 1, 1, 30}, - []uint32{1, 1, 1, 30}, - }, - { - "lags of (30m, 35m, 40m, 45m)", - []uint32{30 * 60, 35 * 60, 40 * 60, 45 * 60}, - []uint32{30 * 60, 35 * 60, 40 * 60, 45 * 60}, - }, - { - "lags of (1s, 1s, 1m, 40m, 40m) - not run filter the second time as first run removed two items.", - []uint32{1, 1, 60, 40 * 60, 40 * 60}, - []uint32{1, 1, 60}, - }, - { - "lags of (1s, 1s, 10m, 40m) - run filter twice to remove two items", - []uint32{1, 1, 10 * 60, 40 * 60}, - []uint32{1, 1}, - }, - { - "lags of (1m, 100m) - return at least 2 items to avoid overloading if the 2nd one is not delayed too much.", - []uint32{1 * 60, 100 * 60}, - []uint32{1 * 60, 100 * 60}, - }, - { - "lags of (1m, 3h) - return 1 if the 2nd one is delayed too much.", - []uint32{1 * 60, 3 * 60 * 60}, - []uint32{1 * 60}, - }, - { - "lags of (3h) - return 1 as they're all delayed too much.", - []uint32{3 * 60 * 60}, - []uint32{3 * 60 * 60}, - }, - { - "lags of (3h, 4h) - return 2 as they're all delayed too much, but still in a good group.", - []uint32{3 * 60 * 60, 4 * 60 * 60}, - []uint32{3 * 60 * 60, 4 * 60 * 60}, - }, - { - "lags of (3h, 3h, 4h) - return 3 as they're all delayed too much, but still in a good group.", - []uint32{3 * 60 * 60, 3 * 60 * 60, 4 * 60 * 60}, - []uint32{3 * 60 * 60, 3 * 60 * 60, 4 * 60 * 60}, - }, - { - "lags of (3h, 15h, 18h) - return 3 as they're all delayed too much, but still in a good group." + - "(different test case than above to show how absurb the good group logic is)", - []uint32{3 * 60 * 60, 15 * 60 * 60, 18 * 60 * 60}, - []uint32{3 * 60 * 60, 15 * 60 * 60, 18 * 60 * 60}, - }, - { - "lags of (3h, 12h, 18h) - return 2 as they're all delayed too much, but 18h is now considered an outlier." + - "(different test case than above to show how absurb the good group logic is)", - []uint32{3 * 60 * 60, 12 * 60 * 60, 18 * 60 * 60}, - []uint32{3 * 60 * 60, 12 * 60 * 60}, - }, - { - "lags of (3h, 30h) - return 2 as they're all delayed too much." + - "(different test case that before, as both tablet stats are" + - "widely different, not within 70% of eachother)", - []uint32{3 * 60 * 60, 30 * 60 * 60}, - []uint32{3 * 60 * 60, 30 * 60 * 60}, - }, - } - - for _, tc := range cases { - lts := make([]*LegacyTabletStats, len(tc.input)) - for i, lag := range tc.input { - lts[i] = &LegacyTabletStats{ - Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), - Serving: true, - Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, - } - } - got := FilterByReplicationLag(lts) - if len(got) != len(tc.output) { - t.Errorf("FilterByReplicationLag(%v) failed: got output:\n%v\nExpected: %v", tc.description, got, tc.output) - continue - } - for i, elag := range tc.output { - if got[i].Stats.SecondsBehindMaster != elag { - t.Errorf("FilterByReplicationLag(%v) failed: got output:\n%v\nExpected value index %v to be %v", tc.description, got, i, elag) - } - } - } -} - func TestFilterByReplicationLagThreeTabletMin(t *testing.T) { // Use at least 3 tablets if possible testSetMinNumTablets(3) // lags of (1s, 1s, 10m, 11m) - returns at least32 items where the slightly delayed ones that are returned are the 10m and 11m ones. - ts1 := &LegacyTabletStats{ + ts1 := &TabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &LegacyTabletStats{ + ts2 := &TabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts3 := &LegacyTabletStats{ + ts3 := &TabletStats{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts4 := &LegacyTabletStats{ + ts4 := &TabletStats{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - got := FilterByReplicationLag([]*LegacyTabletStats{ts1, ts2, ts3, ts4}) + got := FilterStatsByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts1) || !got[1].DeepEqual(ts2) || !got[2].DeepEqual(ts3) { - t.Errorf("FilterByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) + t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } // lags of (11m, 10m, 1s, 1s) - reordered tablets returns the same 3 items where the slightly delayed one that is returned is the 10m and 11m ones. - ts1 = &LegacyTabletStats{ + ts1 = &TabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - ts2 = &LegacyTabletStats{ + ts2 = &TabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts3 = &LegacyTabletStats{ + ts3 = &TabletStats{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts4 = &LegacyTabletStats{ + ts4 = &TabletStats{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - got = FilterByReplicationLag([]*LegacyTabletStats{ts1, ts2, ts3, ts4}) + got = FilterStatsByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts3) || !got[1].DeepEqual(ts4) || !got[2].DeepEqual(ts2) { - t.Errorf("FilterByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) + t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } // Reset to the default testSetMinNumTablets(2) } -func TestFilterByReplicationLagOneTabletMin(t *testing.T) { +func TestFilterStatsByReplicationLagOneTabletMin(t *testing.T) { // Use at least 1 tablets if possible testSetMinNumTablets(1) // lags of (1s, 100m) - return only healthy tablet if that is all that is available. - ts1 := &LegacyTabletStats{ + ts1 := &TabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &LegacyTabletStats{ + ts2 := &TabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got := FilterByReplicationLag([]*LegacyTabletStats{ts1, ts2}) + got := FilterStatsByReplicationLag([]*TabletStats{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { - t.Errorf("FilterByReplicationLag([1s, 100m]) = %+v, want [1s]", got) + t.Errorf("FilterStatsByReplicationLag([1s, 100m]) = %+v, want [1s]", got) } // lags of (1m, 100m) - return only healthy tablet if that is all that is healthy enough. - ts1 = &LegacyTabletStats{ + ts1 = &TabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60}, } - ts2 = &LegacyTabletStats{ + ts2 = &TabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got = FilterByReplicationLag([]*LegacyTabletStats{ts1, ts2}) + got = FilterStatsByReplicationLag([]*TabletStats{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { - t.Errorf("FilterByReplicationLag([1m, 100m]) = %+v, want [1m]", got) + t.Errorf("FilterStatsByReplicationLag([1m, 100m]) = %+v, want [1m]", got) } // Reset to the default testSetMinNumTablets(2) @@ -357,17 +246,17 @@ func TestTrivialStatsUpdate(t *testing.T) { } for _, c := range cases { - o := &LegacyTabletStats{ + o := &TabletStats{ Stats: &querypb.RealtimeStats{ SecondsBehindMaster: c.o, }, } - n := &LegacyTabletStats{ + n := &TabletStats{ Stats: &querypb.RealtimeStats{ SecondsBehindMaster: c.n, }, } - got := TrivialStatsUpdate(o, n) + got := o.TrivialStatsUpdate(n) if got != c.expected { t.Errorf("TrivialStatsUpdate(%v, %v) = %v, expected %v", c.o, c.n, got, c.expected) } diff --git a/go/vt/discovery/tablet_picker.go b/go/vt/discovery/tablet_picker.go index 05e998e1111..2bda68fec42 100644 --- a/go/vt/discovery/tablet_picker.go +++ b/go/vt/discovery/tablet_picker.go @@ -37,7 +37,7 @@ type TabletPicker struct { tabletTypes []topodatapb.TabletType healthCheck LegacyHealthCheck - watcher *TopologyWatcher + watcher *LegacyTopologyWatcher statsCache *LegacyTabletStatsCache } @@ -51,7 +51,7 @@ func NewTabletPicker(ctx context.Context, ts *topo.Server, cell, keyspace, shard // These have to be initialized in the following sequence (watcher must be last). healthCheck := NewLegacyHealthCheck(healthcheckRetryDelay, healthcheckTimeout) statsCache := NewLegacyTabletStatsCache(healthCheck, ts, cell) - watcher := NewShardReplicationWatcher(ctx, ts, healthCheck, cell, keyspace, shard, healthcheckTopologyRefresh, DefaultTopoReadConcurrency) + watcher := NewLegacyShardReplicationWatcher(ctx, ts, healthCheck, cell, keyspace, shard, healthcheckTopologyRefresh, DefaultTopoReadConcurrency) return &TabletPicker{ ts: ts, diff --git a/go/vt/discovery/tablet_stats_cache.go b/go/vt/discovery/tablet_stats_cache.go new file mode 100644 index 00000000000..4bc6dfe6242 --- /dev/null +++ b/go/vt/discovery/tablet_stats_cache.go @@ -0,0 +1,295 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "sync" + + "golang.org/x/net/context" + "vitess.io/vitess/go/vt/log" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/srvtopo" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/topoproto" +) + +// TabletStatsCache is a HealthCheckStatsListener that keeps both the +// current list of available TabletStats, and a serving list: +// - for master tablets, only the current master is kept. +// - for non-master tablets, we filter the list using FilterLegacyStatsByReplicationLag. +// It keeps entries for all tablets in the cell(s) it's configured to serve for, +// and for the master independently of which cell it's in. +// Note the healthy tablet computation is done when we receive a tablet +// update only, not at serving time. +// Also note the cache may not have the last entry received by the tablet. +// For instance, if a tablet was healthy, and is still healthy, we do not +// keep its new update. +type TabletStatsCache struct { + // cell is the cell we are keeping all tablets for. + // Note we keep track of all master tablets in all cells. + cell string + // mu protects the following fields. It does not protect individual + // entries in the entries map. + mu sync.RWMutex + // entries maps from keyspace/shard/tabletType to our cache. + entries map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry + // tsm is a helper to broadcast aggregate stats. + tsm srvtopo.TargetStatsMultiplexer + // cellAliases is a cache of cell aliases + cellAliases map[string]string +} + +// tabletStatsCacheEntry is the per keyspace/shard/tabletType +// entry of the in-memory map for TabletStatsCache. +type tabletStatsCacheEntry struct { + // mu protects the rest of this structure. + mu sync.RWMutex + // all has the valid tablets, indexed by TabletToMapKey(ts.Tablet), + // as it is the index used by HealthCheck. + all map[string]*TabletStats + // healthy only has the healthy ones. + healthy []*TabletStats +} + +func (e *tabletStatsCacheEntry) updateHealthyMapForMaster(ts *TabletStats) { + if ts.Up { + // We have an Up master. + if len(e.healthy) == 0 { + // We have a new Up server, just remember it. + e.healthy = append(e.healthy, ts) + return + } + + // We already have one up server, see if we + // need to replace it. + if ts.TabletExternallyReparentedTimestamp < e.healthy[0].TabletExternallyReparentedTimestamp { + log.Warningf("not marking healthy master %s as Up for %s because its externally reparented timestamp is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", + topoproto.TabletAliasString(ts.Tablet.Alias), + topoproto.KeyspaceShardString(ts.Target.Keyspace, ts.Target.Shard), + topoproto.TabletAliasString(e.healthy[0].Tablet.Alias), + ts.TabletExternallyReparentedTimestamp, + e.healthy[0].TabletExternallyReparentedTimestamp) + return + } + + // Just replace it. + e.healthy[0] = ts + return + } + + // We have a Down master, remove it only if it's exactly the same. + if len(e.healthy) != 0 { + if ts.Key == e.healthy[0].Key { + // Same guy, remove it. + e.healthy = nil + } + } +} + +// NewTabletStatsCache creates a TabletStatsCache, and registers +// it as HealthCheckStatsListener of the provided healthcheck. +// Note we do the registration in this code to guarantee we call +// SetListener with sendDownEvents=true, as we need these events +// to maintain the integrity of our cache. +func NewTabletStatsCache(hc HealthCheck, ts *topo.Server, cell string) *TabletStatsCache { + return newTabletStatsCache(cell) +} + +func newTabletStatsCache(localCell string) *TabletStatsCache { + tc := &TabletStatsCache{ + entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), + tsm: srvtopo.NewTargetStatsMultiplexer(), + cell: localCell, + cellAliases: make(map[string]string), + } + return tc +} + +// UpdateStats is used to ... +func (tc *TabletStatsCache) UpdateStats(ts *TabletStats, topoServer *topo.Server) { + if ts.Target.TabletType != topodatapb.TabletType_MASTER && + ts.Tablet.Alias.Cell != tc.cell && + tc.getAliasByCell(ts.Tablet.Alias.Cell, topoServer) != tc.getAliasByCell(tc.cell, topoServer) { + // this is for a non-master tablet in a different cell and a different alias, drop it + return + } + + e := tc.getOrCreateEntry(ts.Target) + e.mu.Lock() + defer e.mu.Unlock() + + // Update our full map. + trivialNonMasterUpdate := false + if existing, ok := e.all[ts.Key]; ok { + if ts.Up { + // We have an existing entry, and a new entry. + // Remember if they are both good (most common case). + trivialNonMasterUpdate = existing.LastError == nil && existing.Serving && ts.LastError == nil && + ts.Serving && ts.Target.TabletType != topodatapb.TabletType_MASTER && existing.TrivialStatsUpdate(ts) + + // We already have the entry, update the + // values if necessary. (will update both + // 'all' and 'healthy' as they use pointers). + if !trivialNonMasterUpdate { + *existing = *ts + } + } else { + // We have an entry which we shouldn't. Remove it. + delete(e.all, ts.Key) + } + } else { + if ts.Up { + // Add the entry. + e.all[ts.Key] = ts + } else { + // We were told to remove an entry which we + // didn't have anyway, nothing should happen. + return + } + } + + // Update our healthy list. + var allArray []*TabletStats + if ts.Target.TabletType == topodatapb.TabletType_MASTER { + // The healthy list is different for TabletType_MASTER: we + // only keep the most recent one. + e.updateHealthyMapForMaster(ts) + } else { + // For non-master, if it is a trivial update, + // we just skip everything else. We don't even update the + // aggregate stats. + if trivialNonMasterUpdate { + return + } + + // Now we need to do some work. Recompute our healthy list. + allArray = make([]*TabletStats, 0, len(e.all)) + for _, s := range e.all { + allArray = append(allArray, s) + } + e.healthy = FilterStatsByReplicationLag(allArray) + } +} + +// getEntry returns an existing TabletStatsCacheEntry in the cache, or nil +// if the entry does not exist. It only takes a Read lock on mu. +func (tc *TabletStatsCache) getEntry(keyspace, shard string, tabletType topodatapb.TabletType) *tabletStatsCacheEntry { + tc.mu.RLock() + defer tc.mu.RUnlock() + + if s, ok := tc.entries[keyspace]; ok { + if t, ok := s[shard]; ok { + if e, ok := t[tabletType]; ok { + return e + } + } + } + return nil +} + +// getOrCreateEntry returns an existing TabletStatsCacheEntry from the cache, +// or creates it if it doesn't exist. +func (tc *TabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStatsCacheEntry { + // Fast path (most common path too): Read-lock, return the entry. + if e := tc.getEntry(target.Keyspace, target.Shard, target.TabletType); e != nil { + return e + } + + // Slow path: Lock, will probably have to add the entry at some level. + tc.mu.Lock() + defer tc.mu.Unlock() + + s, ok := tc.entries[target.Keyspace] + if !ok { + s = make(map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry) + tc.entries[target.Keyspace] = s + } + t, ok := s[target.Shard] + if !ok { + t = make(map[topodatapb.TabletType]*tabletStatsCacheEntry) + s[target.Shard] = t + } + e, ok := t[target.TabletType] + if !ok { + e = &tabletStatsCacheEntry{ + all: make(map[string]*TabletStats), + } + t[target.TabletType] = e + } + return e +} + +func (tc *TabletStatsCache) getAliasByCell(cell string, topoServer *topo.Server) string { + tc.mu.Lock() + defer tc.mu.Unlock() + + if alias, ok := tc.cellAliases[cell]; ok { + return alias + } + + alias := topo.GetAliasByCell(context.Background(), topoServer, cell) + tc.cellAliases[cell] = alias + + return alias +} + +// GetTabletStats returns the full list of available targets. +// The returned array is owned by the caller. +func (tc *TabletStatsCache) GetTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats { + e := tc.getEntry(keyspace, shard, tabletType) + if e == nil { + return nil + } + + e.mu.RLock() + defer e.mu.RUnlock() + // ok to make a copy here + result := make([]TabletStats, 0, len(e.all)) + for _, s := range e.all { + result = append(result, *s) + } + return result +} + +// GetHealthyTabletStats returns only the healthy targets. +// The returned array is owned by the caller. +// For TabletType_MASTER, this will only return at most one entry, +// the most recent tablet of type master. +func (tc *TabletStatsCache) GetHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats { + e := tc.getEntry(keyspace, shard, tabletType) + if e == nil { + return nil + } + + e.mu.RLock() + defer e.mu.RUnlock() + // ok to make a copy here + result := make([]TabletStats, len(e.healthy)) + for i, ts := range e.healthy { + result[i] = *ts + } + return result +} + +// ResetForTesting is for use in tests only. +func (tc *TabletStatsCache) ResetForTesting() { + tc.mu.Lock() + defer tc.mu.Unlock() + + tc.entries = make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry) +} diff --git a/go/vt/discovery/topology_watcher.go b/go/vt/discovery/topology_watcher.go index f0e55ba9f0d..9648c6dfaae 100644 --- a/go/vt/discovery/topology_watcher.go +++ b/go/vt/discovery/topology_watcher.go @@ -25,11 +25,12 @@ import ( "sync" "time" + "vitess.io/vitess/go/vt/key" + "golang.org/x/net/context" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/trace" - "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" @@ -52,19 +53,11 @@ var ( "Operation", topologyWatcherOpListTablets, topologyWatcherOpGetTablet) ) -// TabletRecorder is the part of the LegacyHealthCheck interface that can -// add or remove tablets. We define it as a sub-interface here so we -// can add filters on tablets if needed. -type TabletRecorder interface { - // AddTablet adds the tablet. - // Name is an alternate name, like an address. - AddTablet(tablet *topodatapb.Tablet, name string) - - // RemoveTablet removes the tablet. - RemoveTablet(tablet *topodatapb.Tablet) - - // ReplaceTablet does an AddTablet and RemoveTablet in one call, effectively replacing the old tablet with the new. - ReplaceTablet(old, new *topodatapb.Tablet, name string) +// tabletInfo is used internally by the TopologyWatcher class +type tabletInfo struct { + alias string + key string + tablet *topodatapb.Tablet } // NewCellTabletsWatcher returns a TopologyWatcher that monitors all @@ -78,7 +71,7 @@ func NewCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, tr Tabl // NewShardReplicationWatcher returns a TopologyWatcher that // monitors the tablets in a cell/keyspace/shard, and starts refreshing. func NewShardReplicationWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) *TopologyWatcher { - return NewTopologyWatcher(ctx, topoServer, tr, cell, refreshInterval, true /* refreshKnownTablets */, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) { + return NewTopologyWatcher(ctx, topoServer, tr, cell, refreshInterval, true /* RefreshKnownTablets */, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) { sri, err := tw.topoServer.GetShardReplication(ctx, tw.cell, keyspace, shard) switch { case err == nil: @@ -98,13 +91,6 @@ func NewShardReplicationWatcher(ctx context.Context, topoServer *topo.Server, tr }) } -// tabletInfo is used internally by the TopologyWatcher class -type tabletInfo struct { - alias string - key string - tablet *topodatapb.Tablet -} - // TopologyWatcher polls tablet from a configurable set of tablets // periodically. When tablets are added / removed, it calls // the TabletRecorder AddTablet / RemoveTablet interface appropriately. @@ -335,12 +321,10 @@ func (tw *TopologyWatcher) TopoChecksum() uint32 { return tw.topoChecksum } -// FilterByShard is a TabletRecorder filter that filters tablets by +// FilterByShard is a filter that filters tablets by // keyspace/shard. type FilterByShard struct { - // tr is the underlying TabletRecorder to forward requests too - tr TabletRecorder - + cell string // filters is a map of keyspace to filters for shards filters map[string][]*filterShard } @@ -358,7 +342,7 @@ type filterShard struct { // can either be a shard name, or a keyrange. All tablets that match // at least one keyspace|shard tuple will be forwarded to the // underlying TabletRecorder. -func NewFilterByShard(tr TabletRecorder, filters []string) (*FilterByShard, error) { +func NewFilterByShard(cell string, filters []string) (*FilterByShard, error) { m := make(map[string][]*filterShard) for _, filter := range filters { parts := strings.Split(filter, "|") @@ -390,35 +374,18 @@ func NewFilterByShard(tr TabletRecorder, filters []string) (*FilterByShard, erro } return &FilterByShard{ - tr: tr, + cell: cell, filters: m, }, nil } -// AddTablet is part of the TabletRecorder interface. -func (fbs *FilterByShard) AddTablet(tablet *topodatapb.Tablet, name string) { - if fbs.isIncluded(tablet) { - fbs.tr.AddTablet(tablet, name) - } -} - -// RemoveTablet is part of the TabletRecorder interface. -func (fbs *FilterByShard) RemoveTablet(tablet *topodatapb.Tablet) { - if fbs.isIncluded(tablet) { - fbs.tr.RemoveTablet(tablet) - } -} - -// ReplaceTablet is part of the TabletRecorder interface. -func (fbs *FilterByShard) ReplaceTablet(old, new *topodatapb.Tablet, name string) { - if fbs.isIncluded(old) && fbs.isIncluded(new) { - fbs.tr.ReplaceTablet(old, new, name) +// IsIncluded returns true iff the tablet's keyspace and shard should be +// forwarded to the underlying TabletRecorder. +func (fbs *FilterByShard) IsIncluded(tablet *topodatapb.Tablet) bool { + if !isTabletInCell(fbs.cell, tablet) { + return false } -} -// isIncluded returns true iff the tablet's keyspace and shard should be -// forwarded to the underlying TabletRecorder. -func (fbs *FilterByShard) isIncluded(tablet *topodatapb.Tablet) bool { canonical, kr, err := topo.ValidateShardName(tablet.Shard) if err != nil { log.Errorf("Error parsing shard name %v, will ignore tablet: %v", tablet.Shard, err) @@ -438,58 +405,48 @@ func (fbs *FilterByShard) isIncluded(tablet *topodatapb.Tablet) bool { return false } -// FilterByKeyspace is a TabletRecorder filter that filters tablets by +// FilterByKeyspace is a filter that filters tablets by // keyspace type FilterByKeyspace struct { - tr TabletRecorder - + cell string keyspaces map[string]bool } -// NewFilterByKeyspace creates a new FilterByKeyspace on top of an existing -// TabletRecorder. Each filter is a keyspace entry. All tablets that match +// NewFilterByKeyspace creates a new FilterByKeyspace. +// Each filter is a keyspace entry. All tablets that match // a keyspace will be forwarded to the underlying TabletRecorder. -func NewFilterByKeyspace(tr TabletRecorder, selectedKeyspaces []string) *FilterByKeyspace { +func NewFilterByKeyspace(cell string, selectedKeyspaces []string) *FilterByKeyspace { m := make(map[string]bool) for _, keyspace := range selectedKeyspaces { m[keyspace] = true } return &FilterByKeyspace{ - tr: tr, + cell: cell, keyspaces: m, } } -// AddTablet is part of the TabletRecorder interface. -func (fbk *FilterByKeyspace) AddTablet(tablet *topodatapb.Tablet, name string) { - if fbk.isIncluded(tablet) { - fbk.tr.AddTablet(tablet, name) - } -} - -// RemoveTablet is part of the TabletRecorder interface. -func (fbk *FilterByKeyspace) RemoveTablet(tablet *topodatapb.Tablet) { - if fbk.isIncluded(tablet) { - fbk.tr.RemoveTablet(tablet) +// IsIncluded returns true if the tablet's keyspace should be +// forwarded to the underlying TabletRecorder. +func (fbk *FilterByKeyspace) IsIncluded(tablet *topodatapb.Tablet) bool { + if !isTabletInCell(fbk.cell, tablet) { + return false } + _, exist := fbk.keyspaces[tablet.Keyspace] + return exist } -// ReplaceTablet is part of the TabletRecorder interface. -func (fbk *FilterByKeyspace) ReplaceTablet(old *topodatapb.Tablet, new *topodatapb.Tablet, name string) { - if old.Keyspace != new.Keyspace { - log.Errorf("Error replacing old tablet in %v with new tablet in %v", old.Keyspace, new.Keyspace) - return +func isTabletInCell(cell string, tablet *topodatapb.Tablet) bool { + if tablet.Type == topodatapb.TabletType_MASTER { + return true } - - if fbk.isIncluded(new) { - fbk.tr.ReplaceTablet(old, new, name) + if tablet.Alias.Cell == cell { + return true } -} - -// isIncluded returns true if the tablet's keyspace should be -// forwarded to the underlying TabletRecorder. -func (fbk *FilterByKeyspace) isIncluded(tablet *topodatapb.Tablet) bool { - _, exist := fbk.keyspaces[tablet.Keyspace] - return exist + // TODO(deepthi): need to implement cell aliases here otherwise they won't work + //if getAliasByCell(tablet.Alias.Cell) == getAliasByCell(cell) { + // return true + //} + return false } diff --git a/go/vt/discovery/utils.go b/go/vt/discovery/utils.go index efebd898348..54858b3a574 100644 --- a/go/vt/discovery/utils.go +++ b/go/vt/discovery/utils.go @@ -26,7 +26,7 @@ import ( // This file contains helper filter methods to process the unfiltered list of // tablets returned by LegacyHealthCheck.GetTabletStatsFrom*. -// See also replicationlag.go for a more sophisicated filter used by vtgate. +// See also legacy_replicationlag.go for a more sophisicated filter used by vtgate. // RemoveUnhealthyTablets filters all unhealthy tablets out. // NOTE: Non-serving tablets are considered healthy. @@ -38,7 +38,7 @@ func RemoveUnhealthyTablets(tabletStatsList []LegacyTabletStats) []LegacyTabletS // source and destination, and the source is not serving (disabled by // TabletControl). When we switch the tablet to 'worker', it will // go back to serving state. - if ts.Stats == nil || ts.Stats.HealthError != "" || ts.LastError != nil || IsReplicationLagHigh(&ts) { + if ts.Stats == nil || ts.Stats.HealthError != "" || ts.LastError != nil || LegacyIsReplicationLagHigh(&ts) { continue } result = append(result, ts) diff --git a/go/vt/schemamanager/schemaswap/schema_swap.go b/go/vt/schemamanager/schemaswap/schema_swap.go index 2f7be38666b..95302ca1e44 100644 --- a/go/vt/schemamanager/schemaswap/schema_swap.go +++ b/go/vt/schemamanager/schemaswap/schema_swap.go @@ -139,7 +139,7 @@ type shardSchemaSwap struct { tabletHealthCheck discovery.LegacyHealthCheck // tabletWatchers contains list of topology watchers monitoring changes in the shard // topology. There are several of them because the watchers are per-cell. - tabletWatchers []*discovery.TopologyWatcher + tabletWatchers []*discovery.LegacyTopologyWatcher // allTabletsLock is a mutex protecting access to contents of health check related // variables below. @@ -697,7 +697,7 @@ func (shardSwap *shardSchemaSwap) startHealthWatchers(ctx context.Context) error return err } for _, cell := range cellList { - watcher := discovery.NewShardReplicationWatcher( + watcher := discovery.NewLegacyShardReplicationWatcher( ctx, topoServer, shardSwap.tabletHealthCheck, @@ -751,7 +751,7 @@ func (shardSwap *shardSchemaSwap) stopHealthWatchers() { // isTabletHealthy verifies that the given LegacyTabletStats represents a healthy tablet that is // caught up with replication to a serving level. func isTabletHealthy(tabletStats *discovery.LegacyTabletStats) bool { - return tabletStats.Stats.HealthError == "" && !discovery.IsReplicationLagHigh(tabletStats) + return tabletStats.Stats.HealthError == "" && !discovery.LegacyIsReplicationLagHigh(tabletStats) } // startWaitingOnUnhealthyTablet registers the tablet as being waited on in a way that diff --git a/go/vt/vtctld/realtime_status.go b/go/vt/vtctld/realtime_status.go index cd0a6f21bf9..43598921a62 100644 --- a/go/vt/vtctld/realtime_status.go +++ b/go/vt/vtctld/realtime_status.go @@ -30,7 +30,7 @@ import ( type realtimeStats struct { healthCheck discovery.LegacyHealthCheck *tabletStatsCache - cellWatchers []*discovery.TopologyWatcher + cellWatchers []*discovery.LegacyTopologyWatcher } func newRealtimeStats(ts *topo.Server) (*realtimeStats, error) { @@ -49,9 +49,9 @@ func newRealtimeStats(ts *topo.Server) (*realtimeStats, error) { if err != nil { return r, fmt.Errorf("error when getting cells: %v", err) } - var watchers []*discovery.TopologyWatcher + var watchers []*discovery.LegacyTopologyWatcher for _, cell := range cells { - watcher := discovery.NewCellTabletsWatcher(context.Background(), ts, hc, cell, *vtctl.HealthCheckTopologyRefresh, true /* refreshKnownTablets */, discovery.DefaultTopoReadConcurrency) + watcher := discovery.NewLegacyCellTabletsWatcher(context.Background(), ts, hc, cell, *vtctl.HealthCheckTopologyRefresh, true /* refreshKnownTablets */, discovery.DefaultTopoReadConcurrency) watchers = append(watchers, watcher) } r.cellWatchers = watchers diff --git a/go/vt/vtgate/api.go b/go/vt/vtgate/api.go index 05f5c85d04d..269da43d903 100644 --- a/go/vt/vtgate/api.go +++ b/go/vt/vtgate/api.go @@ -88,7 +88,63 @@ func getItemPath(url string) string { return parts[1] } -func initAPI(ctx context.Context, hc discovery.LegacyHealthCheck) { +func initAPI(ctx context.Context, hc discovery.HealthCheck) { + // Healthcheck real time status per (cell, keyspace, tablet type, metric). + handleCollection("health-check", func(r *http.Request) (interface{}, error) { + cacheStatus := hc.CacheStatus() + + itemPath := getItemPath(r.URL.Path) + if itemPath == "" { + return cacheStatus, nil + } + parts := strings.SplitN(itemPath, "/", 2) + collectionFilter := parts[0] + if collectionFilter == "" { + return cacheStatus, nil + } + if len(parts) != 2 { + return nil, fmt.Errorf("invalid health-check path: %q expected path: / or /cell/ or /keyspace/ or /tablet/", itemPath) + } + value := parts[1] + + switch collectionFilter { + case "cell": + { + filteredStatus := make(discovery.TabletsCacheStatusList, 0) + for _, tabletCacheStatus := range cacheStatus { + if tabletCacheStatus.Cell == value { + filteredStatus = append(filteredStatus, tabletCacheStatus) + } + } + return filteredStatus, nil + } + case "keyspace": + { + filteredStatus := make(discovery.TabletsCacheStatusList, 0) + for _, tabletCacheStatus := range cacheStatus { + if tabletCacheStatus.Target.Keyspace == value { + filteredStatus = append(filteredStatus, tabletCacheStatus) + } + } + return filteredStatus, nil + } + case "tablet": + { + // Return a _specific tablet_ + for _, tabletCacheStatus := range cacheStatus { + for _, tabletStats := range tabletCacheStatus.TabletsStats { + if tabletStats.Name == value || tabletStats.Tablet.MysqlHostname == value { + return tabletStats, nil + } + } + } + } + } + return nil, fmt.Errorf("cannot find health for: %s", itemPath) + }) +} + +func legacyInitAPI(ctx context.Context, hc discovery.LegacyHealthCheck) { // Healthcheck real time status per (cell, keyspace, tablet type, metric). handleCollection("health-check", func(r *http.Request) (interface{}, error) { cacheStatus := hc.CacheStatus() diff --git a/go/vt/vtgate/discoverygateway.go b/go/vt/vtgate/discoverygateway.go index 118084313e0..ff0ca4c30ac 100644 --- a/go/vt/vtgate/discoverygateway.go +++ b/go/vt/vtgate/discoverygateway.go @@ -17,7 +17,6 @@ limitations under the License. package vtgate import ( - "flag" "fmt" "math/rand" "sort" @@ -28,7 +27,6 @@ import ( "golang.org/x/net/context" "vitess.io/vitess/go/vt/topotools" - "vitess.io/vitess/go/flagutil" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/log" @@ -44,28 +42,17 @@ import ( "vitess.io/vitess/go/vt/topo/topoproto" ) -var ( - cellsToWatch = flag.String("cells_to_watch", "", "comma-separated list of cells for watching tablets") - refreshInterval = flag.Duration("tablet_refresh_interval", 1*time.Minute, "tablet refresh interval") - refreshKnownTablets = flag.Bool("tablet_refresh_known_tablets", true, "tablet refresh reloads the tablet address/port map from topo in case it changes") - topoReadConcurrency = flag.Int("topo_read_concurrency", 32, "concurrent topo reads") - - allowedTabletTypes []topodatapb.TabletType - - tabletFilters flagutil.StringListValue -) - const ( gatewayImplementationDiscovery = "discoverygateway" ) func init() { - flag.Var(&tabletFilters, "tablet_filters", "Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch") - topoproto.TabletTypeListVar(&allowedTabletTypes, "allowed_tablet_types", "Specifies the tablet types this vtgate is allowed to route queries to") RegisterGatewayCreator(gatewayImplementationDiscovery, createDiscoveryGateway) } -type discoveryGateway struct { +// DiscoveryGateway is the default Gateway implementation. +// This implementation uses the legacy healthcheck module. +type DiscoveryGateway struct { queryservice.QueryService hc discovery.LegacyHealthCheck tsc *discovery.LegacyTabletStatsCache @@ -75,7 +62,7 @@ type discoveryGateway struct { // tabletsWatchers contains a list of all the watchers we use. // We create one per cell. - tabletsWatchers []*discovery.TopologyWatcher + tabletsWatchers []*discovery.LegacyTopologyWatcher // mu protects the fields of this group. mu sync.RWMutex @@ -91,11 +78,11 @@ func createDiscoveryGateway(ctx context.Context, hc discovery.LegacyHealthCheck, return NewDiscoveryGateway(ctx, hc, serv, cell, retryCount) } -// NewDiscoveryGateway creates a new discoveryGateway using the provided healthcheck and toposerver. +// NewDiscoveryGateway creates a new DiscoveryGateway using the provided healthcheck and toposerver. // cell is the cell where the gateway is located a.k.a localCell. // This gateway can route to MASTER in any cell provided by the cells_to_watch command line argument. // Other tablet type requests (REPLICA/RDONLY) are only routed to tablets in the same cell. -func NewDiscoveryGateway(ctx context.Context, hc discovery.LegacyHealthCheck, serv srvtopo.Server, cell string, retryCount int) *discoveryGateway { +func NewDiscoveryGateway(ctx context.Context, hc discovery.LegacyHealthCheck, serv srvtopo.Server, cell string, retryCount int) *DiscoveryGateway { var topoServer *topo.Server if serv != nil { var err error @@ -105,13 +92,13 @@ func NewDiscoveryGateway(ctx context.Context, hc discovery.LegacyHealthCheck, se } } - dg := &discoveryGateway{ + dg := &DiscoveryGateway{ hc: hc, tsc: discovery.NewTabletStatsCacheDoNotSetListener(topoServer, cell), srvTopoServer: serv, localCell: cell, retryCount: retryCount, - tabletsWatchers: make([]*discovery.TopologyWatcher, 0, 1), + tabletsWatchers: make([]*discovery.LegacyTopologyWatcher, 0, 1), statusAggregators: make(map[string]*TabletStatusAggregator), buffer: buffer.New(), } @@ -120,27 +107,28 @@ func NewDiscoveryGateway(ctx context.Context, hc discovery.LegacyHealthCheck, se // We set sendDownEvents=true because it's required by LegacyTabletStatsCache. hc.SetListener(dg, true /* sendDownEvents */) - log.Infof("loading tablets for cells: %v", *cellsToWatch) - for _, c := range strings.Split(*cellsToWatch, ",") { + cellsToWatch := *discovery.CellsToWatch + log.Infof("loading tablets for cells: %v", cellsToWatch) + for _, c := range strings.Split(cellsToWatch, ",") { if c == "" { continue } - var tr discovery.TabletRecorder = dg.hc - if len(tabletFilters) > 0 { - if len(KeyspacesToWatch) > 0 { + var recorder discovery.TabletRecorder = dg.hc + if len(discovery.TabletFilters) > 0 { + if len(discovery.KeyspacesToWatch) > 0 { log.Exitf("Only one of -keyspaces_to_watch and -tablet_filters may be specified at a time") } - fbs, err := discovery.NewFilterByShard(dg.hc, tabletFilters) + fbs, err := discovery.NewLegacyFilterByShard(recorder, discovery.TabletFilters) if err != nil { log.Exitf("Cannot parse tablet_filters parameter: %v", err) } - tr = fbs - } else if len(KeyspacesToWatch) > 0 { - tr = discovery.NewFilterByKeyspace(dg.hc, KeyspacesToWatch) + recorder = fbs + } else if len(discovery.KeyspacesToWatch) > 0 { + recorder = discovery.NewLegacyFilterByKeyspace(recorder, discovery.KeyspacesToWatch) } - ctw := discovery.NewCellTabletsWatcher(ctx, topoServer, tr, c, *refreshInterval, *refreshKnownTablets, *topoReadConcurrency) + ctw := discovery.NewLegacyCellTabletsWatcher(ctx, topoServer, recorder, c, *discovery.RefreshInterval, *discovery.RefreshKnownTablets, *discovery.TopoReadConcurrency) dg.tabletsWatchers = append(dg.tabletsWatchers, ctw) } dg.QueryService = queryservice.Wrap(nil, dg.withRetry) @@ -149,7 +137,7 @@ func NewDiscoveryGateway(ctx context.Context, hc discovery.LegacyHealthCheck, se // RegisterStats registers the stats to export the lag since the last refresh // and the checksum of the topology -func (dg *discoveryGateway) RegisterStats() { +func (dg *DiscoveryGateway) RegisterStats() { stats.NewGaugeDurationFunc( "TopologyWatcherMaxRefreshLag", "maximum time since the topology watcher refreshed a cell", @@ -165,7 +153,7 @@ func (dg *discoveryGateway) RegisterStats() { // topologyWatcherMaxRefreshLag returns the maximum lag since the watched // cells were refreshed from the topo server -func (dg *discoveryGateway) topologyWatcherMaxRefreshLag() time.Duration { +func (dg *DiscoveryGateway) topologyWatcherMaxRefreshLag() time.Duration { var lag time.Duration for _, tw := range dg.tabletsWatchers { cellLag := tw.RefreshLag() @@ -177,7 +165,7 @@ func (dg *discoveryGateway) topologyWatcherMaxRefreshLag() time.Duration { } // topologyWatcherChecksum returns a checksum of the topology watcher state -func (dg *discoveryGateway) topologyWatcherChecksum() int64 { +func (dg *DiscoveryGateway) topologyWatcherChecksum() int64 { var checksum int64 for _, tw := range dg.tabletsWatchers { checksum = checksum ^ int64(tw.TopoChecksum()) @@ -187,7 +175,7 @@ func (dg *discoveryGateway) topologyWatcherChecksum() int64 { // StatsUpdate forwards LegacyHealthCheck updates to LegacyTabletStatsCache and MasterBuffer. // It is part of the discovery.LegacyHealthCheckStatsListener interface. -func (dg *discoveryGateway) StatsUpdate(ts *discovery.LegacyTabletStats) { +func (dg *DiscoveryGateway) StatsUpdate(ts *discovery.LegacyTabletStats) { dg.tsc.StatsUpdate(ts) if ts.Target.TabletType == topodatapb.TabletType_MASTER { @@ -196,7 +184,7 @@ func (dg *discoveryGateway) StatsUpdate(ts *discovery.LegacyTabletStats) { } // WaitForTablets is part of the gateway.Gateway interface. -func (dg *discoveryGateway) WaitForTablets(ctx context.Context, tabletTypesToWait []topodatapb.TabletType) error { +func (dg *DiscoveryGateway) WaitForTablets(ctx context.Context, tabletTypesToWait []topodatapb.TabletType) error { // Skip waiting for tablets if we are not told to do so. if len(tabletTypesToWait) == 0 { return nil @@ -213,7 +201,7 @@ func (dg *discoveryGateway) WaitForTablets(ctx context.Context, tabletTypesToWai // Close shuts down underlying connections. // This function hides the inner implementation. -func (dg *discoveryGateway) Close(ctx context.Context) error { +func (dg *DiscoveryGateway) Close(ctx context.Context) error { dg.buffer.Shutdown() for _, ctw := range dg.tabletsWatchers { ctw.Stop() @@ -223,7 +211,7 @@ func (dg *discoveryGateway) Close(ctx context.Context) error { // CacheStatus returns a list of TabletCacheStatus per // keyspace/shard/tablet_type. -func (dg *discoveryGateway) CacheStatus() TabletCacheStatusList { +func (dg *DiscoveryGateway) CacheStatus() TabletCacheStatusList { dg.mu.RLock() res := make(TabletCacheStatusList, 0, len(dg.statusAggregators)) for _, aggr := range dg.statusAggregators { @@ -239,21 +227,21 @@ func (dg *discoveryGateway) CacheStatus() TabletCacheStatusList { // the middle of a transaction. While returning the error check if it maybe a result of // a resharding event, and set the re-resolve bit and let the upper layers // re-resolve and retry. -func (dg *discoveryGateway) withRetry(ctx context.Context, target *querypb.Target, unused queryservice.QueryService, name string, inTransaction bool, inner func(ctx context.Context, target *querypb.Target, conn queryservice.QueryService) (bool, error)) error { +func (dg *DiscoveryGateway) withRetry(ctx context.Context, target *querypb.Target, unused queryservice.QueryService, name string, inTransaction bool, inner func(ctx context.Context, target *querypb.Target, conn queryservice.QueryService) (bool, error)) error { var tabletLastUsed *topodatapb.Tablet var err error invalidTablets := make(map[string]bool) - if len(allowedTabletTypes) > 0 { + if len(discovery.AllowedTabletTypes) > 0 { var match bool - for _, allowed := range allowedTabletTypes { + for _, allowed := range discovery.AllowedTabletTypes { if allowed == target.TabletType { match = true break } } if !match { - return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "requested tablet type %v is not part of the allowed tablet types for this vtgate: %+v", target.TabletType.String(), allowedTabletTypes) + return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "requested tablet type %v is not part of the allowed tablet types for this vtgate: %+v", target.TabletType.String(), discovery.AllowedTabletTypes) } } @@ -378,13 +366,13 @@ func nextTablet(cell string, tablets []discovery.LegacyTabletStats, offset, leng return -1 } -func (dg *discoveryGateway) updateStats(target *querypb.Target, startTime time.Time, err error) { +func (dg *DiscoveryGateway) updateStats(target *querypb.Target, startTime time.Time, err error) { elapsed := time.Since(startTime) aggr := dg.getStatsAggregator(target) aggr.UpdateQueryInfo("", target.TabletType, elapsed, err != nil) } -func (dg *discoveryGateway) getStatsAggregator(target *querypb.Target) *TabletStatusAggregator { +func (dg *DiscoveryGateway) getStatsAggregator(target *querypb.Target) *TabletStatusAggregator { key := fmt.Sprintf("%v/%v/%v", target.Keyspace, target.Shard, target.TabletType.String()) // get existing aggregator diff --git a/go/vt/vtgate/discoverygateway_test.go b/go/vt/vtgate/discoverygateway_test.go index 553a8bfa1f9..96ada8bfe54 100644 --- a/go/vt/vtgate/discoverygateway_test.go +++ b/go/vt/vtgate/discoverygateway_test.go @@ -37,23 +37,23 @@ import ( ) func TestDiscoveryGatewayExecute(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *discoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { _, err := dg.Execute(context.Background(), target, "query", nil, 0, nil) return err }) - testDiscoveryGatewayTransact(t, func(dg *discoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { _, err := dg.Execute(context.Background(), target, "query", nil, 1, nil) return err }) } func TestDiscoveryGatewayExecuteBatch(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *discoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { queries := []*querypb.BoundQuery{{Sql: "query", BindVariables: nil}} _, err := dg.ExecuteBatch(context.Background(), target, queries, false, 0, nil) return err }) - testDiscoveryGatewayTransact(t, func(dg *discoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { queries := []*querypb.BoundQuery{{Sql: "query", BindVariables: nil}} _, err := dg.ExecuteBatch(context.Background(), target, queries, false, 1, nil) return err @@ -61,7 +61,7 @@ func TestDiscoveryGatewayExecuteBatch(t *testing.T) { } func TestDiscoveryGatewayExecuteStream(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *discoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { err := dg.StreamExecute(context.Background(), target, "query", nil, 0, nil, func(qr *sqltypes.Result) error { return nil }) @@ -70,33 +70,33 @@ func TestDiscoveryGatewayExecuteStream(t *testing.T) { } func TestDiscoveryGatewayBegin(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *discoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { _, err := dg.Begin(context.Background(), target, nil) return err }) } func TestDiscoveryGatewayCommit(t *testing.T) { - testDiscoveryGatewayTransact(t, func(dg *discoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { return dg.Commit(context.Background(), target, 1) }) } func TestDiscoveryGatewayRollback(t *testing.T) { - testDiscoveryGatewayTransact(t, func(dg *discoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayTransact(t, func(dg *DiscoveryGateway, target *querypb.Target) error { return dg.Rollback(context.Background(), target, 1) }) } func TestDiscoveryGatewayBeginExecute(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *discoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { _, _, err := dg.BeginExecute(context.Background(), target, "query", nil, nil) return err }) } func TestDiscoveryGatewayBeginExecuteBatch(t *testing.T) { - testDiscoveryGatewayGeneric(t, func(dg *discoveryGateway, target *querypb.Target) error { + testDiscoveryGatewayGeneric(t, func(dg *DiscoveryGateway, target *querypb.Target) error { queries := []*querypb.BoundQuery{{Sql: "query", BindVariables: nil}} _, _, err := dg.BeginExecuteBatch(context.Background(), target, queries, false, nil) return err @@ -264,7 +264,7 @@ func TestDiscoveryGatewayGetTabletsWithRegion(t *testing.T) { } } -func testDiscoveryGatewayGeneric(t *testing.T, f func(dg *discoveryGateway, target *querypb.Target) error) { +func testDiscoveryGatewayGeneric(t *testing.T, f func(dg *DiscoveryGateway, target *querypb.Target) error) { keyspace := "ks" shard := "0" tabletType := topodatapb.TabletType_REPLICA @@ -347,7 +347,7 @@ func testDiscoveryGatewayGeneric(t *testing.T, f func(dg *discoveryGateway, targ } } -func testDiscoveryGatewayTransact(t *testing.T, f func(dg *discoveryGateway, target *querypb.Target) error) { +func testDiscoveryGatewayTransact(t *testing.T, f func(dg *DiscoveryGateway, target *querypb.Target) error) { keyspace := "ks" shard := "0" tabletType := topodatapb.TabletType_REPLICA diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 852b47770ae..07bf0dfcbc8 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -841,7 +841,7 @@ func (e *Executor) handleShow(ctx context.Context, safeSession *SafeSession, sql }, nil case "vitess_tablets": var rows [][]sqltypes.Value - stats := e.scatterConn.GetHealthCheckCacheStatus() + stats := e.scatterConn.GetLegacyHealthCheckCacheStatus() for _, s := range stats { for _, ts := range s.TabletsStats { state := "SERVING" diff --git a/go/vt/vtgate/gateway.go b/go/vt/vtgate/gateway.go index 47d03f8bd50..36cb8f276f8 100644 --- a/go/vt/vtgate/gateway.go +++ b/go/vt/vtgate/gateway.go @@ -18,7 +18,6 @@ import ( "time" "golang.org/x/net/context" - "vitess.io/vitess/go/flagutil" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/discovery" @@ -36,17 +35,8 @@ import ( var ( implementation = flag.String("gateway_implementation", "discoverygateway", "The implementation of gateway") initialTabletTimeout = flag.Duration("gateway_initial_tablet_timeout", 30*time.Second, "At startup, the gateway will wait up to that duration to get one tablet per keyspace/shard/tablettype") - - // KeyspacesToWatch - if provided this specifies which keyspaces should be - // visible to a vtgate. By default the vtgate will allow access to any - // keyspace. - KeyspacesToWatch flagutil.StringListValue ) -func init() { - flag.Var(&KeyspacesToWatch, "keyspaces_to_watch", "Specifies which keyspaces this vtgate should have access to while routing queries or accessing the vschema") -} - // A Gateway is the query processing module for each shard, // which is used by ScatterConn. type Gateway interface { diff --git a/go/vt/vtgate/scatter_conn.go b/go/vt/vtgate/scatter_conn.go index dec03a16d1c..4755d58451e 100644 --- a/go/vt/vtgate/scatter_conn.go +++ b/go/vt/vtgate/scatter_conn.go @@ -89,6 +89,29 @@ func LegacyNewScatterConn(statsName string, txConn *TxConn, gw Gateway, hc disco } } +// NewScatterConn creates a new ScatterConn. +func NewScatterConn(statsName string, txConn *TxConn, gw *TabletGateway) *ScatterConn { + // this only works with TabletGateway + tabletCallErrorCountStatsName := "" + if statsName != "" { + tabletCallErrorCountStatsName = statsName + "ErrorCount" + } + return &ScatterConn{ + timings: stats.NewMultiTimings( + statsName, + "Scatter connection timings", + []string{"Operation", "Keyspace", "ShardName", "DbType"}), + tabletCallErrorCount: stats.NewCountersWithMultiLabels( + tabletCallErrorCountStatsName, + "Error count from tablet calls in scatter conns", + []string{"Operation", "Keyspace", "ShardName", "DbType"}), + txConn: txConn, + gateway: gw, + //TODO(deepthi): we need to get ScatterConn working without using legacyHealthCheck + legacyHealthCheck: nil, + } +} + func (stc *ScatterConn) startAction(name string, target *querypb.Target) (time.Time, []string) { statsKey := []string{name, target.Keyspace, target.Shard, topoproto.TabletTypeLString(target.TabletType)} startTime := time.Now() @@ -419,9 +442,21 @@ func (stc *ScatterConn) GetGatewayCacheStatus() TabletCacheStatusList { return stc.gateway.CacheStatus() } +// GetLegacyHealthCheckCacheStatus returns a displayable version of the HealthCheck cache. +func (stc *ScatterConn) GetLegacyHealthCheckCacheStatus() discovery.LegacyTabletsCacheStatusList { + if stc.legacyHealthCheck != nil { + return stc.legacyHealthCheck.CacheStatus() + } + return nil +} + // GetHealthCheckCacheStatus returns a displayable version of the HealthCheck cache. -func (stc *ScatterConn) GetHealthCheckCacheStatus() discovery.LegacyTabletsCacheStatusList { - return stc.legacyHealthCheck.CacheStatus() +func (stc *ScatterConn) GetHealthCheckCacheStatus() TabletCacheStatusList { + gw, ok := stc.gateway.(*TabletGateway) + if ok { + return gw.CacheStatus() + } + return nil } // multiGo performs the requested 'action' on the specified diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go new file mode 100644 index 00000000000..c4946e47476 --- /dev/null +++ b/go/vt/vtgate/tabletgateway.go @@ -0,0 +1,320 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtgate + +import ( + "fmt" + "math/rand" + "sort" + "sync" + "time" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/vt/discovery" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/srvtopo" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/buffer" + "vitess.io/vitess/go/vt/vttablet/queryservice" + + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" +) + +const ( + tabletGatewayImplementation = "tabletgateway" +) + +func init() { + RegisterGatewayCreator(tabletGatewayImplementation, createTabletGateway) +} + +// TabletGateway implements the Gateway interface. +// This implementation uses the new healthcheck module. +type TabletGateway struct { + queryservice.QueryService + hc discovery.HealthCheck + srvTopoServer srvtopo.Server + localCell string + retryCount int + + // mu protects the fields of this group. + mu sync.Mutex + // statusAggregators is a map indexed by the key + // keyspace/shard/tablet_type. + statusAggregators map[string]*TabletStatusAggregator + + // buffer, if enabled, buffers requests during a detected MASTER failover. + buffer *buffer.Buffer +} + +func createTabletGateway(ctx context.Context, unused discovery.LegacyHealthCheck, serv srvtopo.Server, + cell string, retryCount int) Gateway { + return NewTabletGateway(ctx, serv, cell, retryCount) +} + +// NewTabletGateway creates and returns a new TabletGateway +func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string, retryCount int) *TabletGateway { + var topoServer *topo.Server + if serv != nil { + var err error + topoServer, err = serv.GetTopoServer() + if err != nil { + log.Exitf("Unable to create new TabletGateway: %v", err) + } + } + + hc := discovery.NewHealthCheck(ctx, *HealthCheckRetryDelay, *HealthCheckTimeout, topoServer, localCell) + + gw := &TabletGateway{ + hc: hc, + srvTopoServer: serv, + localCell: localCell, + retryCount: retryCount, + statusAggregators: make(map[string]*TabletStatusAggregator), + buffer: buffer.New(), + } + + // TODO(deepthi): start healthcheck here + //hc.Open() + + gw.QueryService = queryservice.Wrap(nil, gw.withRetry) + return gw +} + +// RegisterStats registers the stats to export the lag since the last refresh +// and the checksum of the topology +func (gw *TabletGateway) RegisterStats() { + gw.hc.RegisterStats() +} + +// StatsUpdate forwards LegacyHealthCheck updates to TabletStatsCache and MasterBuffer. +// It is part of the discovery.HealthCheckStatsListener interface. +// TODO(deepthi): figure out how to update buffer +//func (gw *TabletGateway) StatsUpdate(ts *discovery.LegacyTabletStats) { +// if ts.Target.TabletType == topodatapb.TabletType_MASTER { +// gw.buffer.StatsUpdate(ts) +// } +//} + +// WaitForTablets is part of the Gateway interface. +func (gw *TabletGateway) WaitForTablets(ctx context.Context, tabletTypesToWait []topodatapb.TabletType) error { + // Skip waiting for tablets if we are not told to do so. + if len(tabletTypesToWait) == 0 { + return nil + } + + // Finds the targets to look for. + _, err := srvtopo.FindAllTargets(ctx, gw.srvTopoServer, gw.localCell, tabletTypesToWait) + if err != nil { + return err + } + return nil + //return gw.hc.WaitForAllServingTablets(ctx, targets) +} + +// Close shuts down underlying connections. +// This function hides the inner implementation. +func (gw *TabletGateway) Close(ctx context.Context) error { + gw.buffer.Shutdown() + return gw.hc.Close() +} + +// CacheStatus returns a list of TabletCacheStatus per +// keyspace/shard/tablet_type. +func (gw *TabletGateway) CacheStatus() TabletCacheStatusList { + gw.mu.Lock() + res := make(TabletCacheStatusList, 0, len(gw.statusAggregators)) + for _, aggr := range gw.statusAggregators { + res = append(res, aggr.GetCacheStatus()) + } + gw.mu.Unlock() + sort.Sort(res) + return res +} + +// withRetry gets available connections and executes the action. If there are retryable errors, +// it retries retryCount times before failing. It does not retry if the connection is in +// the middle of a transaction. While returning the error check if it maybe a result of +// a resharding event, and set the re-resolve bit and let the upper layers +// re-resolve and retry. +func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, unused queryservice.QueryService, + name string, inTransaction bool, inner func(ctx context.Context, target *querypb.Target, conn queryservice.QueryService) (bool, error)) error { + var tabletLastUsed *topodatapb.Tablet + var err error + invalidTablets := make(map[string]bool) + + if len(discovery.AllowedTabletTypes) > 0 { + var match bool + for _, allowed := range discovery.AllowedTabletTypes { + if allowed == target.TabletType { + match = true + break + } + } + if !match { + return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, + "requested tablet type %v is not part of the allowed tablet types for this vtgate: %+v", + target.TabletType.String(), discovery.AllowedTabletTypes) + } + } + + bufferedOnce := false + for i := 0; i < gw.retryCount+1; i++ { + // Check if we should buffer MASTER queries which failed due to an ongoing + // failover. + // Note: We only buffer once and only "!inTransaction" queries i.e. + // a) no transaction is necessary (e.g. critical reads) or + // b) no transaction was created yet. + if !bufferedOnce && !inTransaction && target.TabletType == topodatapb.TabletType_MASTER { + // The next call blocks if we should buffer during a failover. + retryDone, bufferErr := gw.buffer.WaitForFailoverEnd(ctx, target.Keyspace, target.Shard, err) + if bufferErr != nil { + // Buffering failed e.g. buffer is already full. Do not retry. + err = vterrors.Errorf( + vterrors.Code(bufferErr), + "failed to automatically buffer and retry failed request during failover: %v original err (type=%T): %v", + bufferErr, err, err) + break + } + + // Request may have been buffered. + if retryDone != nil { + // We're going to retry this request as part of a buffer drain. + // Notify the buffer after we retried. + defer retryDone() + bufferedOnce = true + } + } + + tablets := gw.hc.GetHealthyTabletStats(target.Keyspace, target.Shard, target.TabletType) + if len(tablets) == 0 { + // fail fast if there is no tablet + err = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no valid tablet") + break + } + gw.shuffleTablets(gw.localCell, tablets) + + // skip tablets we tried before + var ts *discovery.TabletStats + for _, t := range tablets { + if _, ok := invalidTablets[t.Key]; !ok { + ts = &t + break + } + } + if ts == nil { + if err == nil { + // do not override error from last attempt. + err = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no available connection") + } + break + } + + // execute + tabletLastUsed = ts.Tablet + conn := gw.hc.GetConnection(ts.Key) + if conn == nil { + err = vterrors.Errorf(vtrpcpb.Code_UNAVAILABLE, "no connection for key %v tablet %+v", ts.Key, ts.Tablet) + invalidTablets[ts.Key] = true + continue + } + + startTime := time.Now() + var canRetry bool + canRetry, err = inner(ctx, ts.Target, conn) + gw.updateStats(target, startTime, err) + if canRetry { + invalidTablets[ts.Key] = true + continue + } + break + } + return NewShardError(err, target, tabletLastUsed) +} + +func (gw *TabletGateway) updateStats(target *querypb.Target, startTime time.Time, err error) { + elapsed := time.Since(startTime) + aggr := gw.getStatsAggregator(target) + aggr.UpdateQueryInfo("", target.TabletType, elapsed, err != nil) +} + +func (gw *TabletGateway) getStatsAggregator(target *querypb.Target) *TabletStatusAggregator { + key := fmt.Sprintf("%v/%v/%v", target.Keyspace, target.Shard, target.TabletType.String()) + + // get existing aggregator + gw.mu.Lock() + defer gw.mu.Unlock() + aggr, ok := gw.statusAggregators[key] + if ok { + return aggr + } + // create a new one if it doesn't exist yet + aggr = NewTabletStatusAggregator(target.Keyspace, target.Shard, target.TabletType, key) + gw.statusAggregators[key] = aggr + return aggr +} + +func (gw *TabletGateway) shuffleTablets(cell string, tablets []discovery.TabletStats) { + sameCell, diffCell, sameCellMax := 0, 0, -1 + length := len(tablets) + + // move all same cell tablets to the front, this is O(n) + for { + sameCellMax = diffCell - 1 + sameCell = gw.nextTablet(cell, tablets, sameCell, length, true) + diffCell = gw.nextTablet(cell, tablets, diffCell, length, false) + // either no more diffs or no more same cells should stop the iteration + if sameCell < 0 || diffCell < 0 { + break + } + + if sameCell < diffCell { + // fast forward the `sameCell` lookup to `diffCell + 1`, `diffCell` unchanged + sameCell = diffCell + 1 + } else { + // sameCell > diffCell, swap needed + tablets[sameCell], tablets[diffCell] = tablets[diffCell], tablets[sameCell] + sameCell++ + diffCell++ + } + } + + //shuffle in same cell tablets + for i := sameCellMax; i > 0; i-- { + swap := rand.Intn(i + 1) + tablets[i], tablets[swap] = tablets[swap], tablets[i] + } + + //shuffle in diff cell tablets + for i, diffCellMin := length-1, sameCellMax+1; i > diffCellMin; i-- { + swap := rand.Intn(i-sameCellMax) + diffCellMin + tablets[i], tablets[swap] = tablets[swap], tablets[i] + } +} + +func (gw *TabletGateway) nextTablet(cell string, tablets []discovery.TabletStats, offset, length int, sameCell bool) int { + for ; offset < length; offset++ { + if (tablets[offset].Tablet.Alias.Cell == cell) == sameCell { + return offset + } + } + return -1 +} diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index c93ce8b6c9f..a647d6f8589 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -60,6 +60,13 @@ var ( _ = flag.Bool("disable_local_gateway", false, "deprecated: if specified, this process will not route any queries to local tablets in the local cell") maxMemoryRows = flag.Int("max_memory_rows", 300000, "Maximum number of rows that will be held in memory for intermediate results as well as the final result.") warnMemoryRows = flag.Int("warn_memory_rows", 30000, "Warning threshold for in-memory results. A row count higher than this amount will cause the VtGateWarnings.ResultsExceeded counter to be incremented.") + + // TODO(deepthi): change these two vars to unexported and move to healthcheck.go when LegacyHealthcheck is removed + + // HealthCheckRetryDelay is the time to wait before retrying healthcheck + HealthCheckRetryDelay = flag.Duration("healthcheck_retry_delay", 2*time.Millisecond, "health check retry delay") + // HealthCheckTimeout is the timeout on the RPC call to tablets + HealthCheckTimeout = flag.Duration("healthcheck_timeout", time.Minute, "the health check timeout period") ) func getTxMode() vtgatepb.TransactionMode { @@ -120,6 +127,91 @@ type RegisterVTGate func(vtgateservice.VTGateService) // RegisterVTGates stores register funcs for VTGate server. var RegisterVTGates []RegisterVTGate +// Init initializes VTGate server. +func Init(ctx context.Context, serv srvtopo.Server, cell string, retryCount int, tabletTypesToWait []topodatapb.TabletType) *VTGate { + if rpcVTGate != nil { + log.Fatalf("VTGate already initialized") + } + + // vschemaCounters needs to be initialized before planner to + // catch the initial load stats. + vschemaCounters = stats.NewCountersWithSingleLabel("VtgateVSchemaCounts", "Vtgate vschema counts", "changes") + + // Build objects from low to high level. + // Start with the gateway. If we can't reach the topology service, + // we can't go on much further, so we log.Fatal out. + gw := NewTabletGateway(ctx, serv, cell, retryCount) + gw.RegisterStats() + if err := WaitForTablets(gw, tabletTypesToWait); err != nil { + log.Fatalf("gateway.WaitForTablets failed: %v", err) + } + + // If we want to filter keyspaces replace the srvtopo.Server with a + // filtering server + if len(discovery.KeyspacesToWatch) > 0 { + log.Infof("Keyspace filtering enabled, selecting %v", discovery.KeyspacesToWatch) + var err error + serv, err = srvtopo.NewKeyspaceFilteringServer(serv, discovery.KeyspacesToWatch) + if err != nil { + log.Fatalf("Unable to construct SrvTopo server: %v", err.Error()) + } + } + + tc := NewTxConn(gw, getTxMode()) + // ScatterConn depends on TxConn to perform forced rollbacks. + sc := NewScatterConn("VttabletCall", tc, gw) + srvResolver := srvtopo.NewResolver(serv, gw, cell) + resolver := NewResolver(srvResolver, serv, cell, sc) + vsm := newVStreamManager(srvResolver, serv, cell) + + rpcVTGate = &VTGate{ + executor: NewExecutor(ctx, serv, cell, resolver, *normalizeQueries, *streamBufferSize, *queryPlanCacheSize), + resolver: resolver, + vsm: vsm, + txConn: tc, + gw: gw, + timings: stats.NewMultiTimings( + "VtgateApi", + "VtgateApi timings", + []string{"Operation", "Keyspace", "DbType"}), + rowsReturned: stats.NewCountersWithMultiLabels( + "VtgateApiRowsReturned", + "Rows returned through the VTgate API", + []string{"Operation", "Keyspace", "DbType"}), + + logExecute: logutil.NewThrottledLogger("Execute", 5*time.Second), + logStreamExecute: logutil.NewThrottledLogger("StreamExecute", 5*time.Second), + } + + errorCounts = stats.NewCountersWithMultiLabels("VtgateApiErrorCounts", "Vtgate API error counts per error type", []string{"Operation", "Keyspace", "DbType", "Code"}) + + _ = stats.NewRates("QPSByOperation", stats.CounterForDimension(rpcVTGate.timings, "Operation"), 15, 1*time.Minute) + _ = stats.NewRates("QPSByKeyspace", stats.CounterForDimension(rpcVTGate.timings, "Keyspace"), 15, 1*time.Minute) + _ = stats.NewRates("QPSByDbType", stats.CounterForDimension(rpcVTGate.timings, "DbType"), 15*60/5, 5*time.Second) + + _ = stats.NewRates("ErrorsByOperation", stats.CounterForDimension(errorCounts, "Operation"), 15, 1*time.Minute) + _ = stats.NewRates("ErrorsByKeyspace", stats.CounterForDimension(errorCounts, "Keyspace"), 15, 1*time.Minute) + _ = stats.NewRates("ErrorsByDbType", stats.CounterForDimension(errorCounts, "DbType"), 15, 1*time.Minute) + _ = stats.NewRates("ErrorsByCode", stats.CounterForDimension(errorCounts, "Code"), 15, 1*time.Minute) + + warnings = stats.NewCountersWithSingleLabel("VtGateWarnings", "Vtgate warnings", "type", "IgnoredSet", "ResultsExceeded") + + servenv.OnRun(func() { + for _, f := range RegisterVTGates { + f(rpcVTGate) + } + }) + rpcVTGate.registerDebugHealthHandler() + err := initQueryLogger(rpcVTGate) + if err != nil { + log.Fatalf("error initializing query logger: %v", err) + } + + initAPI(ctx, gw.hc) + + return rpcVTGate +} + func (vtg *VTGate) registerDebugHealthHandler() { http.HandleFunc("/debug/health", func(w http.ResponseWriter, r *http.Request) { if err := acl.CheckAccessHTTP(r, acl.MONITORING); err != nil { @@ -388,10 +480,10 @@ func LegacyInit(ctx context.Context, hc discovery.LegacyHealthCheck, serv srvtop // If we want to filter keyspaces replace the srvtopo.Server with a // filtering server - if len(KeyspacesToWatch) > 0 { - log.Infof("Keyspace filtering enabled, selecting %v", KeyspacesToWatch) + if len(discovery.KeyspacesToWatch) > 0 { + log.Infof("Keyspace filtering enabled, selecting %v", discovery.KeyspacesToWatch) var err error - serv, err = srvtopo.NewKeyspaceFilteringServer(serv, KeyspacesToWatch) + serv, err = srvtopo.NewKeyspaceFilteringServer(serv, discovery.KeyspacesToWatch) if err != nil { log.Fatalf("Unable to construct SrvTopo server: %v", err.Error()) } @@ -447,7 +539,7 @@ func LegacyInit(ctx context.Context, hc discovery.LegacyHealthCheck, serv srvtop log.Fatalf("error initializing query logger: %v", err) } - initAPI(ctx, hc) + legacyInitAPI(ctx, hc) return rpcVTGate } diff --git a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go index 9178290a1c1..8900d9cfd9d 100644 --- a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go +++ b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go @@ -144,8 +144,8 @@ type ThrottlerInterface interface { } // TopologyWatcherInterface defines the public interface that is implemented by -// discovery.TopologyWatcher. It is only used here to allow mocking out -// go/vt/discovery.TopologyWatcher. +// discovery.LegacyTopologyWatcher. It is only used here to allow mocking out +// go/vt/discovery.LegacyTopologyWatcher. type TopologyWatcherInterface interface { WaitForInitialTopology() error Stop() @@ -182,7 +182,7 @@ func init() { func resetTxThrottlerFactories() { healthCheckFactory = discovery.NewLegacyDefaultHealthCheck topologyWatcherFactory = func(topoServer *topo.Server, tr discovery.TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) TopologyWatcherInterface { - return discovery.NewShardReplicationWatcher(context.Background(), topoServer, tr, cell, keyspace, shard, refreshInterval, topoReadConcurrency) + return discovery.NewLegacyShardReplicationWatcher(context.Background(), topoServer, tr, cell, keyspace, shard, refreshInterval, topoReadConcurrency) } throttlerFactory = func(name, unit string, threadCount int, maxRate, maxReplicationLag int64) (ThrottlerInterface, error) { return throttler.NewThrottler(name, unit, threadCount, maxRate, maxReplicationLag) diff --git a/go/vt/worker/legacy_split_clone.go b/go/vt/worker/legacy_split_clone.go index 46de0ca3d16..0cba9d92ce2 100644 --- a/go/vt/worker/legacy_split_clone.go +++ b/go/vt/worker/legacy_split_clone.go @@ -78,11 +78,11 @@ type LegacySplitCloneWorker struct { // It must be closed at the end of the command. healthCheck discovery.LegacyHealthCheck tsc *discovery.LegacyTabletStatsCache - // destinationShardWatchers contains a TopologyWatcher for each destination + // destinationShardWatchers contains a LegacyTopologyWatcher for each destination // shard. It updates the list of tablets in the healthcheck if replicas are // added/removed. // Each watcher must be stopped at the end of the command. - destinationShardWatchers []*discovery.TopologyWatcher + destinationShardWatchers []*discovery.LegacyTopologyWatcher // destinationDbNames stores for each destination keyspace/shard the MySQL // database name. // Example Map Entry: test_keyspace/-80 => vt_test_keyspace @@ -389,7 +389,7 @@ func (scw *LegacySplitCloneWorker) findTargets(ctx context.Context) error { scw.healthCheck = discovery.NewLegacyHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) scw.tsc = discovery.NewLegacyTabletStatsCache(scw.healthCheck, scw.wr.TopoServer(), scw.cell) for _, si := range scw.destinationShards { - watcher := discovery.NewShardReplicationWatcher(ctx, scw.wr.TopoServer(), scw.healthCheck, + watcher := discovery.NewLegacyShardReplicationWatcher(ctx, scw.wr.TopoServer(), scw.healthCheck, scw.cell, si.Keyspace(), si.ShardName(), *healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) scw.destinationShardWatchers = append(scw.destinationShardWatchers, watcher) diff --git a/go/vt/worker/split_clone.go b/go/vt/worker/split_clone.go index f1d479eae1e..a3422dea6b6 100644 --- a/go/vt/worker/split_clone.go +++ b/go/vt/worker/split_clone.go @@ -105,11 +105,11 @@ type SplitCloneWorker struct { lastPos string // contains the GTID position for the source transactions []int64 - // shardWatchers contains a TopologyWatcher for each source and destination + // shardWatchers contains a LegacyTopologyWatcher for each source and destination // shard. It updates the list of tablets in the healthcheck if replicas are // added/removed. // Each watcher must be stopped at the end of the command. - shardWatchers []*discovery.TopologyWatcher + shardWatchers []*discovery.LegacyTopologyWatcher // destinationDbNames stores for each destination keyspace/shard the MySQL // database name. // Example Map Entry: test_keyspace/-80 => vt_test_keyspace @@ -566,7 +566,7 @@ func (scw *SplitCloneWorker) init(ctx context.Context) error { // Start watchers to get tablets added automatically to healthCheck. allShards := append(scw.sourceShards, scw.destinationShards...) for _, si := range allShards { - watcher := discovery.NewShardReplicationWatcher(ctx, scw.wr.TopoServer(), scw.healthCheck, + watcher := discovery.NewLegacyShardReplicationWatcher(ctx, scw.wr.TopoServer(), scw.healthCheck, scw.cell, si.Keyspace(), si.ShardName(), *healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) scw.shardWatchers = append(scw.shardWatchers, watcher) diff --git a/go/vt/worker/topo_utils.go b/go/vt/worker/topo_utils.go index d63f15705d7..a125910a9eb 100644 --- a/go/vt/worker/topo_utils.go +++ b/go/vt/worker/topo_utils.go @@ -52,7 +52,7 @@ func FindHealthyTablet(ctx context.Context, wr *wrangler.Wrangler, tsc *discover // No healthcheck instance provided. Create one. healthCheck := discovery.NewLegacyHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) tsc = discovery.NewLegacyTabletStatsCache(healthCheck, wr.TopoServer(), cell) - watcher := discovery.NewShardReplicationWatcher(ctx, wr.TopoServer(), healthCheck, cell, keyspace, shard, *healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) + watcher := discovery.NewLegacyShardReplicationWatcher(ctx, wr.TopoServer(), healthCheck, cell, keyspace, shard, *healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) defer watcher.Stop() defer healthCheck.Close() } diff --git a/go/vt/wrangler/keyspace.go b/go/vt/wrangler/keyspace.go index 93293c52d31..56350348dc1 100644 --- a/go/vt/wrangler/keyspace.go +++ b/go/vt/wrangler/keyspace.go @@ -994,7 +994,7 @@ func (wr *Wrangler) waitForDrainInCell(ctx context.Context, cell, keyspace, shar tsc := discovery.NewLegacyTabletStatsCache(hc, wr.TopoServer(), cell) // Create a tablet watcher. - watcher := discovery.NewShardReplicationWatcher(ctx, wr.TopoServer(), hc, cell, keyspace, shard, healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) + watcher := discovery.NewLegacyShardReplicationWatcher(ctx, wr.TopoServer(), hc, cell, keyspace, shard, healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) defer watcher.Stop() // Wait for at least one tablet. From 27ec840c34d2385f10c2f7d2d45d4570f5dea3a9 Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 16 Apr 2020 12:16:25 -0700 Subject: [PATCH 03/39] healthcheck: remove unused structs, compactify new healthcheck function params Signed-off-by: deepthi --- go/cmd/vtgate/vtgate.go | 5 +- go/vt/discovery/healthcheck.go | 22 ++--- go/vt/discovery/legacy_tablet_stats_cache.go | 4 - go/vt/discovery/tablet_stats_cache.go | 12 +-- go/vt/srvtopo/target_stats.go | 87 -------------------- go/vt/vtgate/gateway.go | 3 + go/vt/vtgate/tabletgateway.go | 9 +- go/vt/vtgate/vtgate.go | 4 +- 8 files changed, 28 insertions(+), 118 deletions(-) delete mode 100644 go/vt/srvtopo/target_stats.go diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index 8cb64d04345..7a470f85de2 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -38,7 +38,6 @@ import ( var ( cell = flag.String("cell", "test_nj", "cell to use") - retryCount = flag.Int("retry-count", 2, "retry count") tabletTypesToWait = flag.String("tablet_types_to_wait", "", "wait till connected for specified tablet types during Gateway initialization") useLegacyHealthCheck = flag.Bool("use_legacy_health_check", true, "whether to use the legacy health check") ) @@ -80,9 +79,9 @@ func main() { legacyHealthCheck = discovery.NewLegacyHealthCheck(*vtgate.HealthCheckRetryDelay, *vtgate.HealthCheckTimeout) legacyHealthCheck.RegisterStats() - vtg = vtgate.LegacyInit(context.Background(), legacyHealthCheck, resilientServer, *cell, *retryCount, tabletTypes) + vtg = vtgate.LegacyInit(context.Background(), legacyHealthCheck, resilientServer, *cell, *vtgate.RetryCount, tabletTypes) } else { - vtg = vtgate.Init(context.Background(), resilientServer, *cell, *retryCount, tabletTypes) + vtg = vtgate.Init(context.Background(), resilientServer, *cell, tabletTypes) } servenv.OnRun(func() { diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 4488bfbb4b4..adf7c54ffa2 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -70,8 +70,10 @@ var ( hcErrorCounters = stats.NewCountersWithMultiLabels("HealthcheckErrors", "Healthcheck Errors", []string{"Keyspace", "ShardName", "TabletType"}) hcMasterPromotedCounters = stats.NewCountersWithMultiLabels("HealthcheckMasterPromoted", "Master promoted in keyspace/shard name because of health check errors", []string{"Keyspace", "ShardName"}) healthcheckOnce sync.Once - TabletURLTemplateString = flag.String("tablet_url_template", "http://{{.GetTabletHostPort}}", "format string describing debug tablet url formatting. See the Go code for getTabletDebugURL() how to customize this.") - tabletURLTemplate *template.Template + + // TabletURLTemplateString is a flag to generate URLs for the tablets that vtgate discovers. + TabletURLTemplateString = flag.String("tablet_url_template", "http://{{.GetTabletHostPort}}", "format string describing debug tablet url formatting. See the Go code for getTabletDebugURL() how to customize this.") + tabletURLTemplate *template.Template //TODO(deepthi): change these vars back to unexported when discoveryGateway is removed @@ -178,11 +180,11 @@ type TabletStats struct { Up bool // Serving describes if the tablet can be serving traffic. Serving bool - // TabletExternallyReparentedTimestamp is the last timestamp - // that this tablet was either elected the master, or received + // MasterTermStartTime is the last time at which + // this tablet was either elected the master, or received // a TabletExternallyReparented event. It is set to 0 if the // tablet doesn't think it's a master. - TabletExternallyReparentedTimestamp int64 + MasterTermStartTime int64 // Stats is the current health status, as received by the // StreamHealth RPC (replication lag, ...). Stats *querypb.RealtimeStats @@ -207,7 +209,7 @@ func (e *TabletStats) DeepEqual(f *TabletStats) bool { proto.Equal(e.Target, f.Target) && e.Up == f.Up && e.Serving == f.Serving && - e.TabletExternallyReparentedTimestamp == f.TabletExternallyReparentedTimestamp && + e.MasterTermStartTime == f.MasterTermStartTime && proto.Equal(e.Stats, f.Stats) && ((e.LastError == nil && f.LastError == nil) || (e.LastError != nil && f.LastError != nil && e.LastError.Error() == f.LastError.Error())) @@ -528,7 +530,7 @@ func (hc *HealthCheckImpl) stateChecksum() int64 { ) sort.Sort(st.TabletsStats) for _, ts := range st.TabletsStats { - fmt.Fprintf(&buf, "%v%v%v\n", ts.Up, ts.Serving, ts.TabletExternallyReparentedTimestamp) + fmt.Fprintf(&buf, "%v%v%v\n", ts.Up, ts.Serving, ts.MasterTermStartTime) } } @@ -560,7 +562,7 @@ func (hc *HealthCheckImpl) updateHealth(ts *TabletStats, conn queryservice.Query if oldts.Target.TabletType != topodatapb.TabletType_UNKNOWN && oldts.Target.TabletType != ts.Target.TabletType { // Log and maybe notify log.Infof("HealthCheckUpdate(Type Change): %v, tablet: %s, target %+v => %+v, reparent time: %v", - oldts.Name, topotools.TabletIdent(oldts.Tablet), topotools.TargetIdent(oldts.Target), topotools.TargetIdent(ts.Target), ts.TabletExternallyReparentedTimestamp) + oldts.Name, topotools.TabletIdent(oldts.Tablet), topotools.TargetIdent(oldts.Target), topotools.TargetIdent(ts.Target), ts.MasterTermStartTime) //TODO(deepthi): directly update hc.tsc here //if hc.listener != nil && hc.sendDownEvents { //oldts.Up = false @@ -762,7 +764,7 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St // realtimeStats change. hcc.lastResponseTimestamp = time.Now() hcc.tabletStats.Target = shr.Target - hcc.tabletStats.TabletExternallyReparentedTimestamp = shr.TabletExternallyReparentedTimestamp + hcc.tabletStats.MasterTermStartTime = shr.TabletExternallyReparentedTimestamp hcc.tabletStats.Stats = shr.RealtimeStats hcc.tabletStats.LastError = healthErr reason := "healthCheck update" @@ -931,7 +933,7 @@ func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML { color = "red" extra = " (Down)" } else if ts.Target.TabletType == topodatapb.TabletType_MASTER { - extra = fmt.Sprintf(" (MasterTS: %v)", ts.TabletExternallyReparentedTimestamp) + extra = fmt.Sprintf(" (MasterTS: %v)", ts.MasterTermStartTime) } else { extra = fmt.Sprintf(" (RepLag: %v)", ts.Stats.SecondsBehindMaster) } diff --git a/go/vt/discovery/legacy_tablet_stats_cache.go b/go/vt/discovery/legacy_tablet_stats_cache.go index 2c90efba682..196f3c00b46 100644 --- a/go/vt/discovery/legacy_tablet_stats_cache.go +++ b/go/vt/discovery/legacy_tablet_stats_cache.go @@ -23,7 +23,6 @@ import ( "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" ) @@ -50,8 +49,6 @@ type LegacyTabletStatsCache struct { mu sync.RWMutex // entries maps from keyspace/shard/tabletType to our cache. entries map[string]map[string]map[topodatapb.TabletType]*legacyTabletStatsCacheEntry - // tsm is a helper to broadcast aggregate stats. - tsm srvtopo.TargetStatsMultiplexer // cellAliases is a cache of cell aliases cellAliases map[string]string } @@ -128,7 +125,6 @@ func newLegacyTabletStatsCache(hc LegacyHealthCheck, ts *topo.Server, cell strin cell: cell, ts: ts, entries: make(map[string]map[string]map[topodatapb.TabletType]*legacyTabletStatsCacheEntry), - tsm: srvtopo.NewTargetStatsMultiplexer(), cellAliases: make(map[string]string), } diff --git a/go/vt/discovery/tablet_stats_cache.go b/go/vt/discovery/tablet_stats_cache.go index 4bc6dfe6242..a3e62fc3475 100644 --- a/go/vt/discovery/tablet_stats_cache.go +++ b/go/vt/discovery/tablet_stats_cache.go @@ -23,7 +23,6 @@ import ( "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" ) @@ -31,7 +30,7 @@ import ( // TabletStatsCache is a HealthCheckStatsListener that keeps both the // current list of available TabletStats, and a serving list: // - for master tablets, only the current master is kept. -// - for non-master tablets, we filter the list using FilterLegacyStatsByReplicationLag. +// - for non-master tablets, we filter the list using FilterStatsByReplicationLag. // It keeps entries for all tablets in the cell(s) it's configured to serve for, // and for the master independently of which cell it's in. // Note the healthy tablet computation is done when we receive a tablet @@ -48,8 +47,6 @@ type TabletStatsCache struct { mu sync.RWMutex // entries maps from keyspace/shard/tabletType to our cache. entries map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry - // tsm is a helper to broadcast aggregate stats. - tsm srvtopo.TargetStatsMultiplexer // cellAliases is a cache of cell aliases cellAliases map[string]string } @@ -77,13 +74,13 @@ func (e *tabletStatsCacheEntry) updateHealthyMapForMaster(ts *TabletStats) { // We already have one up server, see if we // need to replace it. - if ts.TabletExternallyReparentedTimestamp < e.healthy[0].TabletExternallyReparentedTimestamp { + if ts.MasterTermStartTime < e.healthy[0].MasterTermStartTime { log.Warningf("not marking healthy master %s as Up for %s because its externally reparented timestamp is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", topoproto.TabletAliasString(ts.Tablet.Alias), topoproto.KeyspaceShardString(ts.Target.Keyspace, ts.Target.Shard), topoproto.TabletAliasString(e.healthy[0].Tablet.Alias), - ts.TabletExternallyReparentedTimestamp, - e.healthy[0].TabletExternallyReparentedTimestamp) + ts.MasterTermStartTime, + e.healthy[0].MasterTermStartTime) return } @@ -113,7 +110,6 @@ func NewTabletStatsCache(hc HealthCheck, ts *topo.Server, cell string) *TabletSt func newTabletStatsCache(localCell string) *TabletStatsCache { tc := &TabletStatsCache{ entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), - tsm: srvtopo.NewTargetStatsMultiplexer(), cell: localCell, cellAliases: make(map[string]string), } diff --git a/go/vt/srvtopo/target_stats.go b/go/vt/srvtopo/target_stats.go deleted file mode 100644 index e492bd59991..00000000000 --- a/go/vt/srvtopo/target_stats.go +++ /dev/null @@ -1,87 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package srvtopo - -import ( - "fmt" - - querypb "vitess.io/vitess/go/vt/proto/query" -) - -// TargetStatsEntry has the updated information for a Target. -type TargetStatsEntry struct { - // Target is what this entry applies to. - Target *querypb.Target - - // TabletExternallyReparentedTimestamp is the latest timestamp - // that was reported for this entry. It applies to masters only. - TabletExternallyReparentedTimestamp int64 -} - -// TargetStatsMultiplexer is a helper class to help broadcast stats updates. -// It doesn't have any synchronization, as the container class will already -// have some and this can just use it. -type TargetStatsMultiplexer struct { - // listeners has the map of channels to send updates to. - listeners map[int]chan (*TargetStatsEntry) - - // nextIndex has the next map id. - nextIndex int -} - -// NewTargetStatsMultiplexer returns an initialized TargetStatsMultiplexer. -func NewTargetStatsMultiplexer() TargetStatsMultiplexer { - return TargetStatsMultiplexer{ - listeners: make(map[int]chan (*TargetStatsEntry)), - } -} - -// Subscribe adds a channel to the list. -// Will change the list. -func (tsm *TargetStatsMultiplexer) Subscribe() (int, <-chan (*TargetStatsEntry)) { - i := tsm.nextIndex - tsm.nextIndex++ - c := make(chan (*TargetStatsEntry), 100) - tsm.listeners[i] = c - return i, c -} - -// Unsubscribe removes a channel from the list. -// Will change the list. -func (tsm *TargetStatsMultiplexer) Unsubscribe(i int) error { - c, ok := tsm.listeners[i] - if !ok { - return fmt.Errorf("TargetStatsMultiplexer.Unsubscribe(%v): not suc channel", i) - } - delete(tsm.listeners, i) - close(c) - return nil -} - -// HasSubscribers returns true if we have registered subscribers. -// Will read the list. -func (tsm *TargetStatsMultiplexer) HasSubscribers() bool { - return len(tsm.listeners) > 0 -} - -// Broadcast sends an update to the list. -// Will read the list. -func (tsm *TargetStatsMultiplexer) Broadcast(tse *TargetStatsEntry) { - for _, c := range tsm.listeners { - c <- tse - } -} diff --git a/go/vt/vtgate/gateway.go b/go/vt/vtgate/gateway.go index 36cb8f276f8..a6b32d14cff 100644 --- a/go/vt/vtgate/gateway.go +++ b/go/vt/vtgate/gateway.go @@ -35,6 +35,9 @@ import ( var ( implementation = flag.String("gateway_implementation", "discoverygateway", "The implementation of gateway") initialTabletTimeout = flag.Duration("gateway_initial_tablet_timeout", 30*time.Second, "At startup, the gateway will wait up to that duration to get one tablet per keyspace/shard/tablettype") + // RetryCount is the number of times a query will be retried on error + // Make this unexported after DiscoveryGateway is deprecated + RetryCount = flag.Int("retry-count", 2, "retry count") ) // A Gateway is the query processing module for each shard, diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index c4946e47476..2e61459d22f 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -66,12 +66,12 @@ type TabletGateway struct { } func createTabletGateway(ctx context.Context, unused discovery.LegacyHealthCheck, serv srvtopo.Server, - cell string, retryCount int) Gateway { - return NewTabletGateway(ctx, serv, cell, retryCount) + cell string, unused2 int) Gateway { + return NewTabletGateway(ctx, serv, cell) } // NewTabletGateway creates and returns a new TabletGateway -func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string, retryCount int) *TabletGateway { +func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string) *TabletGateway { var topoServer *topo.Server if serv != nil { var err error @@ -87,7 +87,7 @@ func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string hc: hc, srvTopoServer: serv, localCell: localCell, - retryCount: retryCount, + retryCount: *RetryCount, statusAggregators: make(map[string]*TabletStatusAggregator), buffer: buffer.New(), } @@ -127,6 +127,7 @@ func (gw *TabletGateway) WaitForTablets(ctx context.Context, tabletTypesToWait [ return err } return nil + // TODO(deepthi): this needs to be implemented //return gw.hc.WaitForAllServingTablets(ctx, targets) } diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index a647d6f8589..bf4560241d4 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -128,7 +128,7 @@ type RegisterVTGate func(vtgateservice.VTGateService) var RegisterVTGates []RegisterVTGate // Init initializes VTGate server. -func Init(ctx context.Context, serv srvtopo.Server, cell string, retryCount int, tabletTypesToWait []topodatapb.TabletType) *VTGate { +func Init(ctx context.Context, serv srvtopo.Server, cell string, tabletTypesToWait []topodatapb.TabletType) *VTGate { if rpcVTGate != nil { log.Fatalf("VTGate already initialized") } @@ -140,7 +140,7 @@ func Init(ctx context.Context, serv srvtopo.Server, cell string, retryCount int, // Build objects from low to high level. // Start with the gateway. If we can't reach the topology service, // we can't go on much further, so we log.Fatal out. - gw := NewTabletGateway(ctx, serv, cell, retryCount) + gw := NewTabletGateway(ctx, serv, cell) gw.RegisterStats() if err := WaitForTablets(gw, tabletTypesToWait); err != nil { log.Fatalf("gateway.WaitForTablets failed: %v", err) From b29b7458624dc6c63a98288a509dcbae049941e6 Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 21 Apr 2020 14:21:48 -0700 Subject: [PATCH 04/39] healthcheck: move tabletStats into separate file, implement cell map of tabletStatsCache, refactor to encapsulate more behavior that belongs in healthcheck Signed-off-by: deepthi --- go/cmd/vtgate/vtgate.go | 5 +- go/vt/discovery/healthcheck.go | 377 ++++++++++++++----------- go/vt/discovery/replicationlag.go | 12 +- go/vt/discovery/replicationlag_test.go | 46 +-- go/vt/discovery/tablet_stats.go | 138 +++++++++ go/vt/discovery/tablet_stats_cache.go | 174 ++---------- go/vt/vtgate/tabletgateway.go | 86 +----- 7 files changed, 410 insertions(+), 428 deletions(-) create mode 100644 go/vt/discovery/tablet_stats.go diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index 7a470f85de2..cbe0e3de23d 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -37,8 +37,9 @@ import ( ) var ( - cell = flag.String("cell", "test_nj", "cell to use") - tabletTypesToWait = flag.String("tablet_types_to_wait", "", "wait till connected for specified tablet types during Gateway initialization") + cell = flag.String("cell", "test_nj", "cell to use") + tabletTypesToWait = flag.String("tablet_types_to_wait", "", "wait till connected for specified tablet types during Gateway initialization") + //TODO(deepthi): remove this and use gateway implementation as the flag. discovery => true, tablet => false useLegacyHealthCheck = flag.Bool("use_legacy_health_check", true, "whether to use the legacy health check") ) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index adf7c54ffa2..0b702410e06 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -27,7 +27,7 @@ limitations under the License. // added or removed from the source. // For a Watcher example have a look at NewShardReplicationWatcher(). // -// TabletStatsCache is one implementation, that caches the known tablets +// tabletStatsCache is one implementation, that caches the known tablets // and the healthy ones per keyspace/shard/tabletType. // // Internally, the HealthCheck module is connected to each tablet and has a @@ -41,18 +41,21 @@ import ( "fmt" "hash/crc32" "html/template" + "math/rand" "net/http" "sort" "strings" "sync" "time" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/flagutil" "vitess.io/vitess/go/vt/topo" "github.com/golang/protobuf/proto" "golang.org/x/net/context" - "vitess.io/vitess/go/netutil" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/sync2" "vitess.io/vitess/go/vt/grpcclient" @@ -126,7 +129,7 @@ const ( Keyspace Shard TabletType - TabletStats + tabletStats {{range $i, $ts := .}} @@ -158,132 +161,6 @@ func init() { flag.Var(&KeyspacesToWatch, "keyspaces_to_watch", "Specifies which keyspaces this vtgate should have access to while routing queries or accessing the vschema") } -// TabletStats is returned when getting the set of tablets. -type TabletStats struct { - // Key uniquely identifies that serving tablet. It is computed - // from the Tablet's record Hostname and PortMap. If a tablet - // is restarted on different ports, its Key will be different. - // Key is computed using the TabletToMapKey method below. - // key can be used in GetConnection(). - Key string - // Tablet is the tablet object that was sent to HealthCheck.AddTablet. - Tablet *topodatapb.Tablet - // Name is an optional tag (e.g. alternative address) for the - // tablet. It is supposed to represent the tablet as a task, - // not as a process. For instance, it can be a - // cell+keyspace+shard+tabletType+taskIndex value. - Name string - // Target is the current target as returned by the streaming - // StreamHealth RPC. - Target *querypb.Target - // Up describes whether the tablet is added or removed. - Up bool - // Serving describes if the tablet can be serving traffic. - Serving bool - // MasterTermStartTime is the last time at which - // this tablet was either elected the master, or received - // a TabletExternallyReparented event. It is set to 0 if the - // tablet doesn't think it's a master. - MasterTermStartTime int64 - // Stats is the current health status, as received by the - // StreamHealth RPC (replication lag, ...). - Stats *querypb.RealtimeStats - // LastError is the error we last saw when trying to get the - // tablet's healthcheck. - LastError error - // TODO(deepthi): No member of this struct should be accessed without holding the mutex - // mu sync.Mutex -} - -// String is defined because we want to print a []*TabletStats array nicely. -func (e *TabletStats) String() string { - return fmt.Sprint(*e) -} - -// DeepEqual compares two TabletStats. Since we include protos, we -// need to use proto.Equal on these. -func (e *TabletStats) DeepEqual(f *TabletStats) bool { - return e.Key == f.Key && - proto.Equal(e.Tablet, f.Tablet) && - e.Name == f.Name && - proto.Equal(e.Target, f.Target) && - e.Up == f.Up && - e.Serving == f.Serving && - e.MasterTermStartTime == f.MasterTermStartTime && - proto.Equal(e.Stats, f.Stats) && - ((e.LastError == nil && f.LastError == nil) || - (e.LastError != nil && f.LastError != nil && e.LastError.Error() == f.LastError.Error())) -} - -// Copy produces a copy of TabletStats. -func (e *TabletStats) Copy() *TabletStats { - ts := *e - return &ts -} - -// GetTabletHostPort formats a tablet host port address. -func (e TabletStats) GetTabletHostPort() string { - vtPort := e.Tablet.PortMap["vt"] - return netutil.JoinHostPort(e.Tablet.Hostname, vtPort) -} - -// GetHostNameLevel returns the specified hostname level. If the level does not exist it will pick the closest level. -// This seems unused but can be utilized by certain url formatting templates. See getTabletDebugURL for more details. -func (e TabletStats) GetHostNameLevel(level int) string { - chunkedHostname := strings.Split(e.Tablet.Hostname, ".") - - if level < 0 { - return chunkedHostname[0] - } else if level >= len(chunkedHostname) { - return chunkedHostname[len(chunkedHostname)-1] - } else { - return chunkedHostname[level] - } -} - -// getTabletDebugURL formats a debug url to the tablet. -// It uses a format string that can be passed into the app to format -// the debug URL to accommodate different network setups. It applies -// the html/template string defined to a TabletStats object. The -// format string can refer to members and functions of TabletStats -// like a regular html/template string. -// -// For instance given a tablet with hostname:port of host.dc.domain:22 -// could be configured as follows: -// http://{{.GetTabletHostPort}} -> http://host.dc.domain:22 -// https://{{.Tablet.Hostname}} -> https://host.dc.domain -// https://{{.GetHostNameLevel 0}}.bastion.corp -> https://host.bastion.corp -func (e TabletStats) getTabletDebugURL() string { - var buffer bytes.Buffer - tabletURLTemplate.Execute(&buffer, e) - return buffer.String() -} - -// TrivialStatsUpdate returns true iff the old and new TabletStats -// haven't changed enough to warrant re-calling FilterLegacyStatsByReplicationLag. -func (e *TabletStats) TrivialStatsUpdate(n *TabletStats) bool { - // Skip replag filter when replag remains in the low rep lag range, - // which should be the case majority of the time. - lowRepLag := lowReplicationLag.Seconds() - oldRepLag := float64(e.Stats.SecondsBehindMaster) - newRepLag := float64(n.Stats.SecondsBehindMaster) - if oldRepLag <= lowRepLag && newRepLag <= lowRepLag { - return true - } - - // Skip replag filter when replag remains in the high rep lag range, - // and did not change beyond +/- 10%. - // when there is a high rep lag, it takes a long time for it to reduce, - // so it is not necessary to re-calculate every time. - // In that case, we won't save the new record, so we still - // remember the original replication lag. - if oldRepLag > lowRepLag && newRepLag > lowRepLag && newRepLag < oldRepLag*1.1 && newRepLag > oldRepLag*0.9 { - return true - } - - return false -} - // TabletRecorder is the part of the HealthCheck interface that can // add or remove tablets. We define it as a sub-interface here so we // can add filters on tablets if needed. @@ -309,7 +186,7 @@ type TabletRecorder interface { // registering a listener. To get the underlying "TabletConn" object // which is used for each tablet, use the "GetConnection()" method // below and pass in the Key string which is also sent to the -// listener in each update (as it is part of TabletStats). +// listener in each update (as it is part of tabletStats). type HealthCheck interface { TabletRecorder // RegisterStats registers the connection counts and checksum stats. @@ -330,8 +207,8 @@ type HealthCheck interface { CacheStatus() TabletsCacheStatusList // Close stops the healthcheck. Close() error - // GetHealthyTabletStats gets the tabletStats by tablet type - GetHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats + // GetTabletAndConnection gets a tablet and connection to execute a query on + GetTabletAndConnection(target *querypb.Target, localCell string, invalidTablets map[string]bool) (string, queryservice.QueryService, error) } type tabletFilterFunc func(tablet *topodatapb.Tablet) bool @@ -347,26 +224,31 @@ type HealthCheckImpl struct { // Immutable fields set at construction time. retryDelay time.Duration healthCheckTimeout time.Duration + ts *topo.Server + cell string + tscByCell map[string]*tabletStatsCache + // connsWG keeps track of all launched Go routines that monitor tablet connections. connsWG sync.WaitGroup // mu protects all the following fields. mu sync.Mutex + // TODO(deepthi): verify all access to following fields is actually being protected by mu + // if not needed, move them up + // addrToHealth maps from address to TabletHealth. addrToHealth map[string]*tabletHealth // Wait group that's used to wait until all initial StatsUpdate() calls are made after the AddTablet() calls. initialUpdatesWG sync.WaitGroup - // ts is the topo server in use. - ts *topo.Server - cell string - tsc *TabletStatsCache - tabletFilters []tabletFilterFunc topoWatchers []*TopologyWatcher + + // cellAliases is a cache of cell aliases + cellAliases map[string]string } // HealthCheckConn is a structure that lives within the scope of @@ -378,7 +260,7 @@ type healthCheckConn struct { ctx context.Context conn queryservice.QueryService - tabletStats TabletStats + tabletStats tabletStats loggedServingState bool lastResponseTimestamp time.Time // timestamp of the last healthcheck response } @@ -392,7 +274,7 @@ type tabletHealth struct { // conn is the connection associated with the tablet. conn queryservice.QueryService // latestTabletStats stores the latest health stats of the tablet. - latestTabletStats TabletStats + latestTabletStats tabletStats } // NewHealthCheck creates a new HealthCheck object. @@ -407,6 +289,8 @@ type tabletHealth struct { func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string) HealthCheck { log.Infof("loading tablets for cells: %v", *CellsToWatch) var filterFuncs []tabletFilterFunc + tscMap := make(map[string]*tabletStatsCache) + tscMap[localCell] = newTabletStatsCache() for _, c := range strings.Split(*CellsToWatch, ",") { if c == "" { @@ -426,6 +310,9 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur fbk := NewFilterByKeyspace(c, KeyspacesToWatch) filterFuncs = append(filterFuncs, fbk.IsIncluded) } + if _, ok := tscMap[c]; !ok { + tscMap[c] = newTabletStatsCache() + } //ctw := NewCellTabletsWatcher(ctx, topoServer, hc, c, *RefreshInterval, // *RefreshKnownTablets, *TopoReadConcurrency) } @@ -436,8 +323,9 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur addrToHealth: make(map[string]*tabletHealth), retryDelay: retryDelay, healthCheckTimeout: healthCheckTimeout, - tsc: newTabletStatsCache(localCell), + tscByCell: tscMap, tabletFilters: filterFuncs, + cellAliases: make(map[string]string), } // create a go func per cell - call watchCell to watch topo and update list of tablets @@ -538,9 +426,9 @@ func (hc *HealthCheckImpl) stateChecksum() int64 { } // updateHealth updates the TabletHealth record and updates the tablet stats -func (hc *HealthCheckImpl) updateHealth(ts *TabletStats, conn queryservice.QueryService) { - // directly update hc.tsc here - hc.tsc.UpdateStats(ts, hc.ts) +func (hc *HealthCheckImpl) updateHealth(ts *tabletStats, conn queryservice.QueryService) { + // update the stats cache + hc.updateStatsCache(ts) hc.mu.Lock() th, ok := hc.addrToHealth[ts.Key] @@ -551,9 +439,8 @@ func (hc *HealthCheckImpl) updateHealth(ts *TabletStats, conn queryservice.Query return } - // TODO(deepthi): do we need to make a copy of the stats? oldts := th.latestTabletStats - th.latestTabletStats = *ts + th.latestTabletStats = *ts.Copy() th.conn = conn hc.mu.Unlock() @@ -563,11 +450,8 @@ func (hc *HealthCheckImpl) updateHealth(ts *TabletStats, conn queryservice.Query // Log and maybe notify log.Infof("HealthCheckUpdate(Type Change): %v, tablet: %s, target %+v => %+v, reparent time: %v", oldts.Name, topotools.TabletIdent(oldts.Tablet), topotools.TargetIdent(oldts.Target), topotools.TargetIdent(ts.Target), ts.MasterTermStartTime) - //TODO(deepthi): directly update hc.tsc here - //if hc.listener != nil && hc.sendDownEvents { - //oldts.Up = false - //hc.listener.StatsUpdate(&oldts) - //} + oldts.Up = false + hc.updateStatsCache(&oldts) // Track how often a tablet gets promoted to master. It is used for // comparing against the variables in go/vtgate/buffer/variables.go. @@ -577,6 +461,75 @@ func (hc *HealthCheckImpl) updateHealth(ts *TabletStats, conn queryservice.Query } } +func (hc *HealthCheckImpl) updateStatsCache(stats *tabletStats) { + if stats.Target.TabletType != topodatapb.TabletType_MASTER && + stats.Tablet.Alias.Cell != hc.cell && + hc.getAliasByCell(stats.Tablet.Alias.Cell, hc.ts) != hc.getAliasByCell(hc.cell, hc.ts) { + // this is for a non-master tablet in a different cell and a different alias, drop it + return + } + + // We assume that if we are getting an update for a tablet, then we are interested in that tablet + // i.e. it belongs to a cell we are watching. + tsc := hc.tscByCell[stats.Tablet.Alias.Cell] + e := tsc.getOrCreateEntry(stats.Target) + e.mu.Lock() + defer e.mu.Unlock() + + // Update our full map. + trivialNonMasterUpdate := false + current, exists := e.all[stats.Key] + + switch { + case exists && stats.Up: + // We have an current entry, and a new entry. + // Remember if they are both good (most common case). + trivialNonMasterUpdate = current.LastError == nil && current.Serving && stats.LastError == nil && + stats.Serving && stats.Target.TabletType != topodatapb.TabletType_MASTER && current.TrivialStatsUpdate(stats) + + // We already have the entry, update the + // values if necessary. (will update both + // 'all' and 'healthy' as they use pointers). + if !trivialNonMasterUpdate { + *current = *stats + } + case exists && !stats.Up: + // We have an entry which we shouldn't. Remove it. + delete(e.all, stats.Key) + case !exists && stats.Up: + // Add the entry. + e.all[stats.Key] = stats + case !exists && !stats.Up: + // We were told to remove an entry which we + // didn't have anyway, nothing should happen. + return + default: + // Unreachable + } + + // Update our healthy list. + var allArray []*tabletStats + if stats.Target.TabletType == topodatapb.TabletType_MASTER { + // The healthy list is different for TabletType_MASTER: we + // only keep the most recent one. + e.updateHealthyMapForMaster(stats) + } else { + // For non-master, if it is a trivial update, + // we just skip everything else. We don't even update the + // aggregate stats. + if trivialNonMasterUpdate { + return + } + + // Now we need to do some work. Recompute our healthy list. + allArray = make([]*tabletStats, 0, len(e.all)) + for _, s := range e.all { + allArray = append(allArray, s) + } + e.healthy = FilterStatsByReplicationLag(allArray) + } +} + // finalizeConn closes the health checking connection and sends the final // notification about the tablet to downstream. To be called only on exit from // checkConn(). @@ -604,7 +557,7 @@ func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn, name string) { // Initial notification for downstream about the tablet existence. // do not copy - hc.updateHealth(hcc.tabletStats.Copy(), hcc.conn) + hc.updateHealth(&hcc.tabletStats, hcc.conn) hc.initialUpdatesWG.Done() retryDelay := hc.retryDelay @@ -748,7 +701,7 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St serving = false } - // hcc.TabletStats.Tablet.Alias.Uid may be 0 because the youtube internal mechanism uses a different + // hcc.tabletStats.Tablet.Alias.Uid may be 0 because the youtube internal mechanism uses a different // code path to initialize this value. If so, we should skip this check. if shr.TabletAlias != nil && hcc.tabletStats.Tablet.Alias.Uid != 0 && !proto.Equal(shr.TabletAlias, hcc.tabletStats.Tablet.Alias) { return fmt.Errorf("health stats mismatch, tablet %+v alias does not match response alias %v", hcc.tabletStats.Tablet, shr.TabletAlias) @@ -811,7 +764,7 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet, name string) { key := TabletToMapKey(tablet) hcc := &healthCheckConn{ ctx: ctx, - tabletStats: TabletStats{ + tabletStats: tabletStats{ Key: key, Tablet: tablet, Name: name, @@ -846,7 +799,7 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet, name string) { hc.connsWG.Add(1) hc.mu.Unlock() - // checkConn should take a TabletStats + // checkConn should take a tabletStats // should exit when healthcheck context is canceled go hc.checkConn(hcc, name) } @@ -889,7 +842,7 @@ type TabletsCacheStatus struct { } // TabletStatsList is used for sorting. -type TabletStatsList []*TabletStats +type TabletStatsList []*tabletStats // Len is part of sort.Interface. func (tsl TabletStatsList) Len() int { @@ -1037,21 +990,117 @@ func (hc *HealthCheckImpl) topologyWatcherChecksum() int64 { return checksum } +// GetTabletAndConnection gets you a tablet connection and it's "Key" as produced by TabletToMapKey +// The Key is used by the caller to keep track of invalidTablets +func (hc *HealthCheckImpl) GetTabletAndConnection(target *querypb.Target, localCell string, invalidTablets map[string]bool) (string, queryservice.QueryService, error) { + tablets := hc.getHealthyTabletStats(target.Keyspace, target.Shard, target.TabletType) + if len(tablets) == 0 { + // fail fast if there is no tablet + err := vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no valid tablet") + return "", nil, err + } + hc.shuffleTablets(localCell, tablets) + + // skip tablets we tried before + var ts *tabletStats + for _, t := range tablets { + if _, ok := invalidTablets[t.Key]; !ok { + ts = &t + conn := hc.GetConnection(ts.Key) + if conn == nil { + invalidTablets[ts.Key] = true + } else { + return ts.Key, conn, nil + } + } + } + err := vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no available connection") + return "", nil, err +} + // GetHealthyTabletStats returns only the healthy targets. // The returned array is owned by the caller. // For TabletType_MASTER, this will only return at most one entry, // the most recent tablet of type master. -func (hc *HealthCheckImpl) GetHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats { - e := hc.tsc.getEntry(keyspace, shard, tabletType) - if e == nil { - return nil +func (hc *HealthCheckImpl) getHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []tabletStats { + var result []tabletStats + // we check all tablet types in all cells because of cellAliases + for _, tsc := range hc.tscByCell { + e := tsc.getEntry(keyspace, shard, tabletType) + if e != nil { + result = append(result, e.getHealthyTabletStats()...) + } } + return result +} - e.mu.RLock() - defer e.mu.RUnlock() - result := make([]TabletStats, len(e.healthy)) - for i, ts := range e.healthy { - result[i] = *ts +func (e *tabletStatsCacheEntry) getHealthyTabletStats() []tabletStats { + e.mu.Lock() + defer e.mu.Unlock() + result := make([]tabletStats, len(e.healthy)) + for _, ts := range e.healthy { + result = append(result, *ts) } return result } + +func (hc *HealthCheckImpl) shuffleTablets(cell string, tablets []tabletStats) { + sameCell, diffCell, sameCellMax := 0, 0, -1 + length := len(tablets) + + // move all same cell tablets to the front, this is O(n) + for { + sameCellMax = diffCell - 1 + sameCell = hc.nextTablet(cell, tablets, sameCell, length, true) + diffCell = hc.nextTablet(cell, tablets, diffCell, length, false) + // either no more diffs or no more same cells should stop the iteration + if sameCell < 0 || diffCell < 0 { + break + } + + if sameCell < diffCell { + // fast forward the `sameCell` lookup to `diffCell + 1`, `diffCell` unchanged + sameCell = diffCell + 1 + } else { + // sameCell > diffCell, swap needed + tablets[sameCell], tablets[diffCell] = tablets[diffCell], tablets[sameCell] + sameCell++ + diffCell++ + } + } + + //shuffle in same cell tablets + for i := sameCellMax; i > 0; i-- { + swap := rand.Intn(i + 1) + tablets[i], tablets[swap] = tablets[swap], tablets[i] + } + + //shuffle in diff cell tablets + for i, diffCellMin := length-1, sameCellMax+1; i > diffCellMin; i-- { + swap := rand.Intn(i-sameCellMax) + diffCellMin + tablets[i], tablets[swap] = tablets[swap], tablets[i] + } +} + +func (hc *HealthCheckImpl) nextTablet(cell string, tablets []tabletStats, offset, length int, sameCell bool) int { + for ; offset < length; offset++ { + if (tablets[offset].Tablet.Alias.Cell == cell) == sameCell { + return offset + } + } + return -1 +} + +func (hc *HealthCheckImpl) getAliasByCell(cell string, topoServer *topo.Server) string { + hc.mu.Lock() + defer hc.mu.Unlock() + + if alias, ok := hc.cellAliases[cell]; ok { + return alias + } + + alias := topo.GetAliasByCell(context.Background(), topoServer, cell) + hc.cellAliases[cell] = alias + + return alias +} diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index 8c5eae415fd..ec4ffaa01ce 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -31,13 +31,13 @@ var ( // IsReplicationLagHigh verifies that the given LegacyTabletStats refers to a tablet with high // replication lag, i.e. higher than the configured discovery_low_replication_lag flag. -func IsReplicationLagHigh(tabletStats *TabletStats) bool { +func IsReplicationLagHigh(tabletStats *tabletStats) bool { return float64(tabletStats.Stats.SecondsBehindMaster) > lowReplicationLag.Seconds() } // IsReplicationLagVeryHigh verifies that the given LegacyTabletStats refers to a tablet with very high // replication lag, i.e. higher than the configured discovery_high_replication_lag_minimum_serving flag. -func IsReplicationLagVeryHigh(tabletStats *TabletStats) bool { +func IsReplicationLagVeryHigh(tabletStats *tabletStats) bool { return float64(tabletStats.Stats.SecondsBehindMaster) > highReplicationLagMinServing.Seconds() } @@ -66,11 +66,11 @@ func IsReplicationLagVeryHigh(tabletStats *TabletStats) bool { // The default for this is 2h, same as the discovery_high_replication_lag_minimum_serving here. // * degraded_threshold: this is only used by vttablet for display. It should match // discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. -func FilterStatsByReplicationLag(tabletStatsList []*TabletStats) []*TabletStats { +func FilterStatsByReplicationLag(tabletStatsList []*tabletStats) []*tabletStats { return filterStatsByLag(tabletStatsList) } -func filterStatsByLag(tabletStatsList []*TabletStats) []*TabletStats { +func filterStatsByLag(tabletStatsList []*tabletStats) []*tabletStats { list := make([]tabletLagSnapshot, 0, len(tabletStatsList)) // filter non-serving tablets and those with very high replication lag for _, ts := range tabletStatsList { @@ -87,7 +87,7 @@ func filterStatsByLag(tabletStatsList []*TabletStats) []*TabletStats { sort.Sort(tabletLagSnapshotList(list)) // Pick those with low replication lag, but at least minNumTablets tablets regardless. - res := make([]*TabletStats, 0, len(list)) + res := make([]*tabletStats, 0, len(list)) for i := 0; i < len(list); i++ { if !IsReplicationLagHigh(list[i].ts) || i < *minNumTablets { res = append(res, list[i].ts) @@ -97,7 +97,7 @@ func filterStatsByLag(tabletStatsList []*TabletStats) []*TabletStats { } type tabletLagSnapshot struct { - ts *TabletStats + ts *tabletStats replag uint32 } type tabletLagSnapshotList []tabletLagSnapshot diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index 40b001aaa1b..57a64ef521d 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -31,17 +31,17 @@ func testSetMinNumTablets(newMin int) { func TestFilterByReplicationLagUnhealthy(t *testing.T) { // 1 healthy serving tablet, 1 not healhty - ts1 := &TabletStats{ + ts1 := &tabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{}, } - ts2 := &TabletStats{ + ts2 := &tabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: false, Stats: &querypb.RealtimeStats{}, } - got := FilterStatsByReplicationLag([]*TabletStats{ts1, ts2}) + got := FilterStatsByReplicationLag([]*tabletStats{ts1, ts2}) if len(got) != 1 { t.Errorf("len(FilterStatsByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}])) = %v, want 1", len(got)) } @@ -102,9 +102,9 @@ func TestFilterByReplicationLag(t *testing.T) { } for _, tc := range cases { - lts := make([]*TabletStats, len(tc.input)) + lts := make([]*tabletStats, len(tc.input)) for i, lag := range tc.input { - lts[i] = &TabletStats{ + lts[i] = &tabletStats{ Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, @@ -130,52 +130,52 @@ func TestFilterByReplicationLagThreeTabletMin(t *testing.T) { // Use at least 3 tablets if possible testSetMinNumTablets(3) // lags of (1s, 1s, 10m, 11m) - returns at least32 items where the slightly delayed ones that are returned are the 10m and 11m ones. - ts1 := &TabletStats{ + ts1 := &tabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &TabletStats{ + ts2 := &tabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts3 := &TabletStats{ + ts3 := &tabletStats{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts4 := &TabletStats{ + ts4 := &tabletStats{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - got := FilterStatsByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4}) + got := FilterStatsByReplicationLag([]*tabletStats{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts1) || !got[1].DeepEqual(ts2) || !got[2].DeepEqual(ts3) { t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } // lags of (11m, 10m, 1s, 1s) - reordered tablets returns the same 3 items where the slightly delayed one that is returned is the 10m and 11m ones. - ts1 = &TabletStats{ + ts1 = &tabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - ts2 = &TabletStats{ + ts2 = &tabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts3 = &TabletStats{ + ts3 = &tabletStats{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts4 = &TabletStats{ + ts4 = &tabletStats{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - got = FilterStatsByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4}) + got = FilterStatsByReplicationLag([]*tabletStats{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts3) || !got[1].DeepEqual(ts4) || !got[2].DeepEqual(ts2) { t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } @@ -187,32 +187,32 @@ func TestFilterStatsByReplicationLagOneTabletMin(t *testing.T) { // Use at least 1 tablets if possible testSetMinNumTablets(1) // lags of (1s, 100m) - return only healthy tablet if that is all that is available. - ts1 := &TabletStats{ + ts1 := &tabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &TabletStats{ + ts2 := &tabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got := FilterStatsByReplicationLag([]*TabletStats{ts1, ts2}) + got := FilterStatsByReplicationLag([]*tabletStats{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { t.Errorf("FilterStatsByReplicationLag([1s, 100m]) = %+v, want [1s]", got) } // lags of (1m, 100m) - return only healthy tablet if that is all that is healthy enough. - ts1 = &TabletStats{ + ts1 = &tabletStats{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60}, } - ts2 = &TabletStats{ + ts2 = &tabletStats{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got = FilterStatsByReplicationLag([]*TabletStats{ts1, ts2}) + got = FilterStatsByReplicationLag([]*tabletStats{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { t.Errorf("FilterStatsByReplicationLag([1m, 100m]) = %+v, want [1m]", got) } @@ -246,12 +246,12 @@ func TestTrivialStatsUpdate(t *testing.T) { } for _, c := range cases { - o := &TabletStats{ + o := &tabletStats{ Stats: &querypb.RealtimeStats{ SecondsBehindMaster: c.o, }, } - n := &TabletStats{ + n := &tabletStats{ Stats: &querypb.RealtimeStats{ SecondsBehindMaster: c.n, }, diff --git a/go/vt/discovery/tablet_stats.go b/go/vt/discovery/tablet_stats.go new file mode 100644 index 00000000000..fc72577271f --- /dev/null +++ b/go/vt/discovery/tablet_stats.go @@ -0,0 +1,138 @@ +package discovery + +import ( + "bytes" + "fmt" + "strings" + + "github.com/golang/protobuf/proto" + "vitess.io/vitess/go/netutil" + "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/proto/topodata" +) + +// tabletStats is returned when getting the set of tablets. +type tabletStats struct { + // Key uniquely identifies that serving tablet. It is computed + // from the Tablet's record Hostname and PortMap. If a tablet + // is restarted on different ports, its Key will be different. + // Key is computed using the TabletToMapKey method below. + // key can be used in GetConnection(). + Key string + // Tablet is the tablet object that was sent to HealthCheck.AddTablet. + Tablet *topodata.Tablet + // Name is an optional tag (e.g. alternative address) for the + // tablet. It is supposed to represent the tablet as a task, + // not as a process. For instance, it can be a + // cell+keyspace+shard+tabletType+taskIndex value. + Name string + // Target is the current target as returned by the streaming + // StreamHealth RPC. + Target *query.Target + // Up describes whether the tablet is added or removed. + Up bool + // Serving describes if the tablet can be serving traffic. + Serving bool + // MasterTermStartTime is the last time at which + // this tablet was either elected the master, or received + // a TabletExternallyReparented event. It is set to 0 if the + // tablet doesn't think it's a master. + MasterTermStartTime int64 + // Stats is the current health status, as received by the + // StreamHealth RPC (replication lag, ...). + Stats *query.RealtimeStats + // LastError is the error we last saw when trying to get the + // tablet's healthcheck. + LastError error + // TODO(deepthi): No member of this struct should be accessed without holding the mutex + // mu sync.Mutex +} + +// String is defined because we want to print a []*tabletStats array nicely. +func (e *tabletStats) String() string { + return fmt.Sprint(*e) +} + +// DeepEqual compares two tabletStats. Since we include protos, we +// need to use proto.Equal on these. +func (e *tabletStats) DeepEqual(f *tabletStats) bool { + return e.Key == f.Key && + proto.Equal(e.Tablet, f.Tablet) && + e.Name == f.Name && + proto.Equal(e.Target, f.Target) && + e.Up == f.Up && + e.Serving == f.Serving && + e.MasterTermStartTime == f.MasterTermStartTime && + proto.Equal(e.Stats, f.Stats) && + ((e.LastError == nil && f.LastError == nil) || + (e.LastError != nil && f.LastError != nil && e.LastError.Error() == f.LastError.Error())) +} + +// Copy produces a copy of tabletStats. +func (e *tabletStats) Copy() *tabletStats { + ts := *e + return &ts +} + +// GetTabletHostPort formats a tablet host port address. +func (e tabletStats) GetTabletHostPort() string { + vtPort := e.Tablet.PortMap["vt"] + return netutil.JoinHostPort(e.Tablet.Hostname, vtPort) +} + +// GetHostNameLevel returns the specified hostname level. If the level does not exist it will pick the closest level. +// This seems unused but can be utilized by certain url formatting templates. See getTabletDebugURL for more details. +func (e tabletStats) GetHostNameLevel(level int) string { + chunkedHostname := strings.Split(e.Tablet.Hostname, ".") + + if level < 0 { + return chunkedHostname[0] + } else if level >= len(chunkedHostname) { + return chunkedHostname[len(chunkedHostname)-1] + } else { + return chunkedHostname[level] + } +} + +// getTabletDebugURL formats a debug url to the tablet. +// It uses a format string that can be passed into the app to format +// the debug URL to accommodate different network setups. It applies +// the html/template string defined to a tabletStats object. The +// format string can refer to members and functions of tabletStats +// like a regular html/template string. +// +// For instance given a tablet with hostname:port of host.dc.domain:22 +// could be configured as follows: +// http://{{.GetTabletHostPort}} -> http://host.dc.domain:22 +// https://{{.Tablet.Hostname}} -> https://host.dc.domain +// https://{{.GetHostNameLevel 0}}.bastion.corp -> https://host.bastion.corp +func (e tabletStats) getTabletDebugURL() string { + var buffer bytes.Buffer + tabletURLTemplate.Execute(&buffer, e) + return buffer.String() +} + +// TrivialStatsUpdate returns true iff the old and new tabletStats +// haven't changed enough to warrant re-calling FilterLegacyStatsByReplicationLag. +func (e *tabletStats) TrivialStatsUpdate(n *tabletStats) bool { + // Skip replag filter when replag remains in the low rep lag range, + // which should be the case majority of the time. + lowRepLag := lowReplicationLag.Seconds() + oldRepLag := float64(e.Stats.SecondsBehindMaster) + newRepLag := float64(n.Stats.SecondsBehindMaster) + if oldRepLag <= lowRepLag && newRepLag <= lowRepLag { + return true + } + + // Skip replag filter when replag remains in the high rep lag range, + // and did not change beyond +/- 10%. + // when there is a high rep lag, it takes a long time for it to reduce, + // so it is not necessary to re-calculate every time. + // In that case, we won't save the new record, so we still + // remember the original replication lag. + if oldRepLag > lowRepLag && newRepLag > lowRepLag && newRepLag < oldRepLag*1.1 && newRepLag > oldRepLag*0.9 { + return true + } + + return false +} diff --git a/go/vt/discovery/tablet_stats_cache.go b/go/vt/discovery/tablet_stats_cache.go index a3e62fc3475..41bd2209b2f 100644 --- a/go/vt/discovery/tablet_stats_cache.go +++ b/go/vt/discovery/tablet_stats_cache.go @@ -19,51 +19,44 @@ package discovery import ( "sync" - "golang.org/x/net/context" "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" ) -// TabletStatsCache is a HealthCheckStatsListener that keeps both the -// current list of available TabletStats, and a serving list: +// tabletStatsCache is an internal data structure that keeps both the +// current list of available tabletStats, and a serving list: // - for master tablets, only the current master is kept. // - for non-master tablets, we filter the list using FilterStatsByReplicationLag. // It keeps entries for all tablets in the cell(s) it's configured to serve for, // and for the master independently of which cell it's in. // Note the healthy tablet computation is done when we receive a tablet // update only, not at serving time. -// Also note the cache may not have the last entry received by the tablet. +// Also note the cache may not have the last entry received from the tablet. // For instance, if a tablet was healthy, and is still healthy, we do not // keep its new update. -type TabletStatsCache struct { - // cell is the cell we are keeping all tablets for. - // Note we keep track of all master tablets in all cells. - cell string +type tabletStatsCache struct { // mu protects the following fields. It does not protect individual // entries in the entries map. - mu sync.RWMutex + mu sync.Mutex // entries maps from keyspace/shard/tabletType to our cache. entries map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry - // cellAliases is a cache of cell aliases - cellAliases map[string]string } // tabletStatsCacheEntry is the per keyspace/shard/tabletType -// entry of the in-memory map for TabletStatsCache. +// entry of the in-memory map for tabletStatsCache. type tabletStatsCacheEntry struct { // mu protects the rest of this structure. - mu sync.RWMutex + mu sync.Mutex // all has the valid tablets, indexed by TabletToMapKey(ts.Tablet), // as it is the index used by HealthCheck. - all map[string]*TabletStats + all map[string]*tabletStats // healthy only has the healthy ones. - healthy []*TabletStats + healthy []*tabletStats } -func (e *tabletStatsCacheEntry) updateHealthyMapForMaster(ts *TabletStats) { +func (e *tabletStatsCacheEntry) updateHealthyMapForMaster(ts *tabletStats) { if ts.Up { // We have an Up master. if len(e.healthy) == 0 { @@ -98,95 +91,18 @@ func (e *tabletStatsCacheEntry) updateHealthyMapForMaster(ts *TabletStats) { } } -// NewTabletStatsCache creates a TabletStatsCache, and registers -// it as HealthCheckStatsListener of the provided healthcheck. -// Note we do the registration in this code to guarantee we call -// SetListener with sendDownEvents=true, as we need these events -// to maintain the integrity of our cache. -func NewTabletStatsCache(hc HealthCheck, ts *topo.Server, cell string) *TabletStatsCache { - return newTabletStatsCache(cell) -} - -func newTabletStatsCache(localCell string) *TabletStatsCache { - tc := &TabletStatsCache{ - entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), - cell: localCell, - cellAliases: make(map[string]string), +func newTabletStatsCache() *tabletStatsCache { + tc := &tabletStatsCache{ + entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), } return tc } -// UpdateStats is used to ... -func (tc *TabletStatsCache) UpdateStats(ts *TabletStats, topoServer *topo.Server) { - if ts.Target.TabletType != topodatapb.TabletType_MASTER && - ts.Tablet.Alias.Cell != tc.cell && - tc.getAliasByCell(ts.Tablet.Alias.Cell, topoServer) != tc.getAliasByCell(tc.cell, topoServer) { - // this is for a non-master tablet in a different cell and a different alias, drop it - return - } - - e := tc.getOrCreateEntry(ts.Target) - e.mu.Lock() - defer e.mu.Unlock() - - // Update our full map. - trivialNonMasterUpdate := false - if existing, ok := e.all[ts.Key]; ok { - if ts.Up { - // We have an existing entry, and a new entry. - // Remember if they are both good (most common case). - trivialNonMasterUpdate = existing.LastError == nil && existing.Serving && ts.LastError == nil && - ts.Serving && ts.Target.TabletType != topodatapb.TabletType_MASTER && existing.TrivialStatsUpdate(ts) - - // We already have the entry, update the - // values if necessary. (will update both - // 'all' and 'healthy' as they use pointers). - if !trivialNonMasterUpdate { - *existing = *ts - } - } else { - // We have an entry which we shouldn't. Remove it. - delete(e.all, ts.Key) - } - } else { - if ts.Up { - // Add the entry. - e.all[ts.Key] = ts - } else { - // We were told to remove an entry which we - // didn't have anyway, nothing should happen. - return - } - } - - // Update our healthy list. - var allArray []*TabletStats - if ts.Target.TabletType == topodatapb.TabletType_MASTER { - // The healthy list is different for TabletType_MASTER: we - // only keep the most recent one. - e.updateHealthyMapForMaster(ts) - } else { - // For non-master, if it is a trivial update, - // we just skip everything else. We don't even update the - // aggregate stats. - if trivialNonMasterUpdate { - return - } - - // Now we need to do some work. Recompute our healthy list. - allArray = make([]*TabletStats, 0, len(e.all)) - for _, s := range e.all { - allArray = append(allArray, s) - } - e.healthy = FilterStatsByReplicationLag(allArray) - } -} - // getEntry returns an existing TabletStatsCacheEntry in the cache, or nil // if the entry does not exist. It only takes a Read lock on mu. -func (tc *TabletStatsCache) getEntry(keyspace, shard string, tabletType topodatapb.TabletType) *tabletStatsCacheEntry { - tc.mu.RLock() - defer tc.mu.RUnlock() +func (tc *tabletStatsCache) getEntry(keyspace, shard string, tabletType topodatapb.TabletType) *tabletStatsCacheEntry { + tc.mu.Lock() + defer tc.mu.Unlock() if s, ok := tc.entries[keyspace]; ok { if t, ok := s[shard]; ok { @@ -200,7 +116,7 @@ func (tc *TabletStatsCache) getEntry(keyspace, shard string, tabletType topodata // getOrCreateEntry returns an existing TabletStatsCacheEntry from the cache, // or creates it if it doesn't exist. -func (tc *TabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStatsCacheEntry { +func (tc *tabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStatsCacheEntry { // Fast path (most common path too): Read-lock, return the entry. if e := tc.getEntry(target.Keyspace, target.Shard, target.TabletType); e != nil { return e @@ -223,67 +139,15 @@ func (tc *TabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStat e, ok := t[target.TabletType] if !ok { e = &tabletStatsCacheEntry{ - all: make(map[string]*TabletStats), + all: make(map[string]*tabletStats), } t[target.TabletType] = e } return e } -func (tc *TabletStatsCache) getAliasByCell(cell string, topoServer *topo.Server) string { - tc.mu.Lock() - defer tc.mu.Unlock() - - if alias, ok := tc.cellAliases[cell]; ok { - return alias - } - - alias := topo.GetAliasByCell(context.Background(), topoServer, cell) - tc.cellAliases[cell] = alias - - return alias -} - -// GetTabletStats returns the full list of available targets. -// The returned array is owned by the caller. -func (tc *TabletStatsCache) GetTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats { - e := tc.getEntry(keyspace, shard, tabletType) - if e == nil { - return nil - } - - e.mu.RLock() - defer e.mu.RUnlock() - // ok to make a copy here - result := make([]TabletStats, 0, len(e.all)) - for _, s := range e.all { - result = append(result, *s) - } - return result -} - -// GetHealthyTabletStats returns only the healthy targets. -// The returned array is owned by the caller. -// For TabletType_MASTER, this will only return at most one entry, -// the most recent tablet of type master. -func (tc *TabletStatsCache) GetHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats { - e := tc.getEntry(keyspace, shard, tabletType) - if e == nil { - return nil - } - - e.mu.RLock() - defer e.mu.RUnlock() - // ok to make a copy here - result := make([]TabletStats, len(e.healthy)) - for i, ts := range e.healthy { - result[i] = *ts - } - return result -} - // ResetForTesting is for use in tests only. -func (tc *TabletStatsCache) ResetForTesting() { +func (tc *tabletStatsCache) ResetForTesting() { tc.mu.Lock() defer tc.mu.Unlock() diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index 2e61459d22f..33055032674 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -18,7 +18,6 @@ package vtgate import ( "fmt" - "math/rand" "sort" "sync" "time" @@ -105,7 +104,7 @@ func (gw *TabletGateway) RegisterStats() { gw.hc.RegisterStats() } -// StatsUpdate forwards LegacyHealthCheck updates to TabletStatsCache and MasterBuffer. +// StatsUpdate forwards HealthCheck updates to TabletStatsCache and MasterBuffer. // It is part of the discovery.HealthCheckStatsListener interface. // TODO(deepthi): figure out how to update buffer //func (gw *TabletGateway) StatsUpdate(ts *discovery.LegacyTabletStats) { @@ -205,45 +204,23 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, } } - tablets := gw.hc.GetHealthyTabletStats(target.Keyspace, target.Shard, target.TabletType) - if len(tablets) == 0 { - // fail fast if there is no tablet - err = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no valid tablet") - break - } - gw.shuffleTablets(gw.localCell, tablets) - - // skip tablets we tried before - var ts *discovery.TabletStats - for _, t := range tablets { - if _, ok := invalidTablets[t.Key]; !ok { - ts = &t - break - } - } - if ts == nil { - if err == nil { - // do not override error from last attempt. - err = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no available connection") - } + tabletLastUsed, conn, connErr := gw.hc.GetTabletAndConnection(target, gw.localCell, invalidTablets) + // execute + if connErr != nil { + err = connErr break } - - // execute - tabletLastUsed = ts.Tablet - conn := gw.hc.GetConnection(ts.Key) if conn == nil { - err = vterrors.Errorf(vtrpcpb.Code_UNAVAILABLE, "no connection for key %v tablet %+v", ts.Key, ts.Tablet) - invalidTablets[ts.Key] = true + err = vterrors.Errorf(vtrpcpb.Code_UNAVAILABLE, "no connection for target %v on attempt #%v", target, i+1) continue } startTime := time.Now() var canRetry bool - canRetry, err = inner(ctx, ts.Target, conn) + canRetry, err = inner(ctx, target, conn) gw.updateStats(target, startTime, err) if canRetry { - invalidTablets[ts.Key] = true + invalidTablets[tabletLastUsed] = true continue } break @@ -272,50 +249,3 @@ func (gw *TabletGateway) getStatsAggregator(target *querypb.Target) *TabletStatu gw.statusAggregators[key] = aggr return aggr } - -func (gw *TabletGateway) shuffleTablets(cell string, tablets []discovery.TabletStats) { - sameCell, diffCell, sameCellMax := 0, 0, -1 - length := len(tablets) - - // move all same cell tablets to the front, this is O(n) - for { - sameCellMax = diffCell - 1 - sameCell = gw.nextTablet(cell, tablets, sameCell, length, true) - diffCell = gw.nextTablet(cell, tablets, diffCell, length, false) - // either no more diffs or no more same cells should stop the iteration - if sameCell < 0 || diffCell < 0 { - break - } - - if sameCell < diffCell { - // fast forward the `sameCell` lookup to `diffCell + 1`, `diffCell` unchanged - sameCell = diffCell + 1 - } else { - // sameCell > diffCell, swap needed - tablets[sameCell], tablets[diffCell] = tablets[diffCell], tablets[sameCell] - sameCell++ - diffCell++ - } - } - - //shuffle in same cell tablets - for i := sameCellMax; i > 0; i-- { - swap := rand.Intn(i + 1) - tablets[i], tablets[swap] = tablets[swap], tablets[i] - } - - //shuffle in diff cell tablets - for i, diffCellMin := length-1, sameCellMax+1; i > diffCellMin; i-- { - swap := rand.Intn(i-sameCellMax) + diffCellMin - tablets[i], tablets[swap] = tablets[swap], tablets[i] - } -} - -func (gw *TabletGateway) nextTablet(cell string, tablets []discovery.TabletStats, offset, length int, sameCell bool) int { - for ; offset < length; offset++ { - if (tablets[offset].Tablet.Alias.Cell == cell) == sameCell { - return offset - } - } - return -1 -} From 9e3adee47354b88243be8f81d3d8de2ead876cf4 Mon Sep 17 00:00:00 2001 From: deepthi Date: Wed, 22 Apr 2020 16:37:57 -0700 Subject: [PATCH 05/39] healthcheck: implement topo watch, tablet_stats_cache test, stub new healthcheck tests Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 323 +++++++++++++++--- go/vt/discovery/healthcheck_test.go | 206 +++++++++++ .../legacy_healthcheck_flaky_test.go | 16 +- .../legacy_tablet_stats_cache_test.go | 2 +- go/vt/discovery/tablet_stats_cache.go | 120 ++++--- go/vt/discovery/tablet_stats_cache_test.go | 92 +++++ go/vt/discovery/topology_watcher.go | 217 +----------- go/vt/vtgate/discoverygateway.go | 6 +- go/vt/vtgate/tabletgateway.go | 13 +- 9 files changed, 671 insertions(+), 324 deletions(-) create mode 100644 go/vt/discovery/healthcheck_test.go create mode 100644 go/vt/discovery/tablet_stats_cache_test.go diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 0b702410e06..ea63f8258ed 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -48,10 +48,11 @@ import ( "sync" "time" + "vitess.io/vitess/go/flagutil" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/flagutil" "vitess.io/vitess/go/vt/topo" "github.com/golang/protobuf/proto" @@ -80,12 +81,6 @@ var ( //TODO(deepthi): change these vars back to unexported when discoveryGateway is removed - // RefreshInterval is the interval at which healthcheck refreshes its list of tablets from topo - RefreshInterval = flag.Duration("tablet_refresh_interval", 1*time.Minute, "tablet refresh interval") - // RefreshKnownTablets tells us whether to process all tablets or only new tablets - RefreshKnownTablets = flag.Bool("tablet_refresh_known_tablets", true, "tablet refresh reloads the tablet address/port map from topo in case it changes") - // TopoReadConcurrency tells us how many topo reads are allowed in parallel - TopoReadConcurrency = flag.Int("topo_read_concurrency", 32, "concurrent topo reads") // CellsToWatch is the list of cells this healthcheck operates over CellsToWatch = flag.String("cells_to_watch", "", "comma-separated list of cells for watching tablets") // AllowedTabletTypes is the list of allowed tablet types. e.g. {MASTER, REPLICA} @@ -96,6 +91,12 @@ var ( // visible to a vtgate. By default the vtgate will allow access to any // keyspace. KeyspacesToWatch flagutil.StringListValue + // RefreshInterval is the interval at which healthcheck refreshes its list of tablets from topo + RefreshInterval = flag.Duration("tablet_refresh_interval", 1*time.Minute, "tablet refresh interval") + // RefreshKnownTablets tells us whether to process all tablets or only new tablets + RefreshKnownTablets = flag.Bool("tablet_refresh_known_tablets", true, "tablet refresh reloads the tablet address/port map from topo in case it changes") + // TopoReadConcurrency tells us how many topo reads are allowed in parallel + TopoReadConcurrency = flag.Int("topo_read_concurrency", 32, "concurrent topo reads") ) // See the documentation for NewLegacyHealthCheck below for an explanation of these parameters. @@ -181,12 +182,6 @@ type TabletRecorder interface { // to a lot of tablets. Tablets are added / removed by calling the // AddTablet / RemoveTablet methods (other discovery module objects // can for instance watch the topology and call these). -// -// Updates to the health of all registered tablet can be watched by -// registering a listener. To get the underlying "TabletConn" object -// which is used for each tablet, use the "GetConnection()" method -// below and pass in the Key string which is also sent to the -// listener in each update (as it is part of tabletStats). type HealthCheck interface { TabletRecorder // RegisterStats registers the connection counts and checksum stats. @@ -201,18 +196,18 @@ type HealthCheck interface { // method. WaitForInitialStatsUpdates won't wait for StatsUpdate() calls // corresponding to AddTablet() calls made during its execution. WaitForInitialStatsUpdates() - // GetConnection returns the TabletConn of the given tablet. - GetConnection(key string) queryservice.QueryService // CacheStatus returns a displayable version of the cache. CacheStatus() TabletsCacheStatusList + // Open starts the healthcheck + Open() // Close stops the healthcheck. Close() error // GetTabletAndConnection gets a tablet and connection to execute a query on GetTabletAndConnection(target *querypb.Target, localCell string, invalidTablets map[string]bool) (string, queryservice.QueryService, error) + // WaitForAllServingTablets + WaitForAllServingTablets(ctx context.Context, targets []*querypb.Target) error } -type tabletFilterFunc func(tablet *topodatapb.Tablet) bool - // HealthCheckImpl performs health checking and notifies downstream components about any changes. // It contains a map of TabletHealth objects, each of which stores the health information for // a tablet. A checkConn goroutine is spawned for each TabletHealth, which is responsible for @@ -226,6 +221,7 @@ type HealthCheckImpl struct { healthCheckTimeout time.Duration ts *topo.Server cell string + cellsToWatch []string tscByCell map[string]*tabletStatsCache // connsWG keeps track of all launched Go routines that monitor tablet connections. @@ -243,8 +239,6 @@ type HealthCheckImpl struct { // Wait group that's used to wait until all initial StatsUpdate() calls are made after the AddTablet() calls. initialUpdatesWG sync.WaitGroup - tabletFilters []tabletFilterFunc - topoWatchers []*TopologyWatcher // cellAliases is a cache of cell aliases @@ -255,7 +249,6 @@ type HealthCheckImpl struct { // the checkConn goroutine to maintain its internal state. Therefore, // it does not require synchronization. Changes that are relevant to // healthcheck are transmitted through calls to HealthCheckImpl.updateHealth. -// TODO(deepthi): replace with goroutine (already using a goroutine to update this) type healthCheckConn struct { ctx context.Context @@ -288,48 +281,49 @@ type tabletHealth struct { // not healthy. func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string) HealthCheck { log.Infof("loading tablets for cells: %v", *CellsToWatch) - var filterFuncs []tabletFilterFunc tscMap := make(map[string]*tabletStatsCache) tscMap[localCell] = newTabletStatsCache() + var topoWatchers []*TopologyWatcher + var allCells = []string{localCell} + var filter TabletFilter for _, c := range strings.Split(*CellsToWatch, ",") { if c == "" { continue } + // very simplistic, assumes localCell is not given as part of *CellsToWatch + allCells = append(allCells, c) + if _, ok := tscMap[c]; !ok { + tscMap[c] = newTabletStatsCache() + } if len(TabletFilters) > 0 { if len(KeyspacesToWatch) > 0 { log.Exitf("Only one of -keyspaces_to_watch and -tablet_filters may be specified at a time") } - fbs, err := NewFilterByShard(c, TabletFilters) + fbs, err := NewFilterByShard(TabletFilters) if err != nil { log.Exitf("Cannot parse tablet_filters parameter: %v", err) } - filterFuncs = append(filterFuncs, fbs.IsIncluded) + filter = fbs } else if len(KeyspacesToWatch) > 0 { - fbk := NewFilterByKeyspace(c, KeyspacesToWatch) - filterFuncs = append(filterFuncs, fbk.IsIncluded) - } - if _, ok := tscMap[c]; !ok { - tscMap[c] = newTabletStatsCache() + filter = NewFilterByKeyspace(c, KeyspacesToWatch) } - //ctw := NewCellTabletsWatcher(ctx, topoServer, hc, c, *RefreshInterval, - // *RefreshKnownTablets, *TopoReadConcurrency) + topoWatchers = append(topoWatchers, NewCellTabletsWatcher(ctx, topoServer, filter, c, *RefreshInterval, *RefreshKnownTablets, *TopoReadConcurrency)) } hc := &HealthCheckImpl{ ts: topoServer, cell: localCell, + cellsToWatch: allCells, addrToHealth: make(map[string]*tabletHealth), retryDelay: retryDelay, healthCheckTimeout: healthCheckTimeout, tscByCell: tscMap, - tabletFilters: filterFuncs, cellAliases: make(map[string]string), + topoWatchers: topoWatchers, } - // create a go func per cell - call watchCell to watch topo and update list of tablets - healthcheckOnce.Do(func() { http.Handle("/debug/gateway", hc) }) @@ -337,14 +331,156 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur return hc } -// TODO(deepthi): implement the watch -//func watchCell(ctx context.Context, topoServer *topo.Server, hc *HealthCheck, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) { -// -//} +// Open starts the healthcheck +func (hc *HealthCheckImpl) Open() { + // start the topo watches here + for _, tw := range hc.topoWatchers { + go hc.watchTopo(tw) + } +} + +func (hc *HealthCheckImpl) watchTopo(tw *TopologyWatcher) { + tw.wg.Add(1) + defer tw.wg.Done() + ticker := time.NewTicker(tw.refreshInterval) + defer ticker.Stop() + for { + hc.loadTablets(tw) + select { + case <-tw.ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (hc *HealthCheckImpl) loadTablets(tw *TopologyWatcher) { + var wg sync.WaitGroup + newTablets := make(map[string]*tabletInfo) + replacedTablets := make(map[string]*tabletInfo) + + tabletAliases, err := tw.getTablets(tw) + topologyWatcherOperations.Add(topologyWatcherOpListTablets, 1) + if err != nil { + topologyWatcherErrors.Add(topologyWatcherOpListTablets, 1) + select { + case <-tw.ctx.Done(): + return + default: + } + log.Errorf("cannot get tablets for cell: %v: %v", tw.cell, err) + return + } + + // Accumulate a list of all known alias strings to use later + // when sorting + tabletAliasStrs := make([]string, 0, len(tabletAliases)) + + tw.mu.Lock() + for _, tAlias := range tabletAliases { + aliasStr := topoproto.TabletAliasString(tAlias) + tabletAliasStrs = append(tabletAliasStrs, aliasStr) + + if !tw.refreshKnownTablets { + if val, ok := tw.tablets[aliasStr]; ok { + newTablets[aliasStr] = val + continue + } + } + + wg.Add(1) + go func(alias *topodatapb.TabletAlias) { + defer wg.Done() + tw.sem <- 1 // Wait for active queue to drain. + tablet, err := tw.topoServer.GetTablet(tw.ctx, alias) + topologyWatcherOperations.Add(topologyWatcherOpGetTablet, 1) + <-tw.sem // Done; enable next request to run + if err != nil { + topologyWatcherErrors.Add(topologyWatcherOpGetTablet, 1) + select { + case <-tw.ctx.Done(): + return + default: + } + log.Errorf("cannot get tablet for alias %v: %v", alias, err) + return + } + if !(hc.isTabletInCell(tablet.Tablet) && (tw.tabletFilter == nil || tw.tabletFilter.IsIncluded(tablet.Tablet))) { + return + } + tw.mu.Lock() + aliasStr := topoproto.TabletAliasString(alias) + newTablets[aliasStr] = &tabletInfo{ + alias: aliasStr, + key: TabletToMapKey(tablet.Tablet), + tablet: tablet.Tablet, + } + tw.mu.Unlock() + }(tAlias) + } + + tw.mu.Unlock() + wg.Wait() + tw.mu.Lock() + + for alias, newVal := range newTablets { + if val, ok := tw.tablets[alias]; !ok { + // Check if there's a tablet with the same address key but a + // different alias. If so, replace it and keep track of the + // replaced alias to make sure it isn't removed later. + found := false + for _, otherVal := range tw.tablets { + if newVal.key == otherVal.key { + found = true + hc.ReplaceTablet(otherVal.tablet, newVal.tablet, alias) + topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) + replacedTablets[otherVal.alias] = newVal + } + } + if !found { + hc.AddTablet(newVal.tablet, alias) + topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1) + } + + } else if val.key != newVal.key { + // Handle the case where the same tablet alias is now reporting + // a different address key. + replacedTablets[alias] = newVal + hc.ReplaceTablet(val.tablet, newVal.tablet, alias) + topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) + } + } + + for _, val := range tw.tablets { + if _, ok := newTablets[val.alias]; !ok { + if _, ok2 := replacedTablets[val.alias]; !ok2 { + hc.RemoveTablet(val.tablet) + topologyWatcherOperations.Add(topologyWatcherOpRemoveTablet, 1) + } + } + } + tw.tablets = newTablets + if !tw.firstLoadDone { + tw.firstLoadDone = true + close(tw.firstLoadChan) + } + + // iterate through the tablets in a stable order and compute a + // checksum of the tablet map + sort.Strings(tabletAliasStrs) + var buf bytes.Buffer + for _, alias := range tabletAliasStrs { + tabletInfo, ok := tw.tablets[alias] + if ok { + buf.WriteString(alias) + buf.WriteString(tabletInfo.key) + } + } + tw.topoChecksum = crc32.ChecksumIEEE(buf.Bytes()) + tw.lastRefresh = time.Now() + + tw.mu.Unlock() -// Open starts healthcheck -func (*HealthCheckImpl) Open() { - // create a local cancelable context, cancel it in Close } // RegisterStats registers the connection counts stats @@ -464,7 +600,7 @@ func (hc *HealthCheckImpl) updateHealth(ts *tabletStats, conn queryservice.Query func (hc *HealthCheckImpl) updateStatsCache(stats *tabletStats) { if stats.Target.TabletType != topodatapb.TabletType_MASTER && stats.Tablet.Alias.Cell != hc.cell && - hc.getAliasByCell(stats.Tablet.Alias.Cell, hc.ts) != hc.getAliasByCell(hc.cell, hc.ts) { + hc.getAliasByCell(stats.Tablet.Alias.Cell) != hc.getAliasByCell(hc.cell) { // this is for a non-master tablet in a different cell and a different alias, drop it return } @@ -822,8 +958,8 @@ func (hc *HealthCheckImpl) WaitForInitialStatsUpdates() { hc.initialUpdatesWG.Wait() } -// GetConnection returns the TabletConn of the given tablet. -func (hc *HealthCheckImpl) GetConnection(key string) queryservice.QueryService { +// getConnection returns the TabletConn of the given tablet. +func (hc *HealthCheckImpl) getConnection(key string) queryservice.QueryService { hc.mu.Lock() defer hc.mu.Unlock() @@ -957,6 +1093,9 @@ func (hc *HealthCheckImpl) Close() error { th.cancelFunc() } hc.addrToHealth = nil + for _, tw := range hc.topoWatchers { + tw.Stop() + } // Release the lock early or a pending checkHealthCheckTimeout // cannot get a read lock on it. hc.mu.Unlock() @@ -1006,7 +1145,7 @@ func (hc *HealthCheckImpl) GetTabletAndConnection(target *querypb.Target, localC for _, t := range tablets { if _, ok := invalidTablets[t.Key]; !ok { ts = &t - conn := hc.GetConnection(ts.Key) + conn := hc.getConnection(ts.Key) if conn == nil { invalidTablets[ts.Key] = true } else { @@ -1034,12 +1173,18 @@ func (hc *HealthCheckImpl) getHealthyTabletStats(keyspace, shard string, tabletT return result } -func (e *tabletStatsCacheEntry) getHealthyTabletStats() []tabletStats { - e.mu.Lock() - defer e.mu.Unlock() - result := make([]tabletStats, len(e.healthy)) - for _, ts := range e.healthy { - result = append(result, *ts) +// GetHealthyTabletStats returns only the healthy targets. +// The returned array is owned by the caller. +// For TabletType_MASTER, this will only return at most one entry, +// the most recent tablet of type master. +func (hc *HealthCheckImpl) getTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []tabletStats { + var result []tabletStats + // we check all tablet types in all cells because of cellAliases + for _, tsc := range hc.tscByCell { + e := tsc.getEntry(keyspace, shard, tabletType) + if e != nil { + result = append(result, e.getTabletStats()...) + } } return result } @@ -1091,7 +1236,7 @@ func (hc *HealthCheckImpl) nextTablet(cell string, tablets []tabletStats, offset return -1 } -func (hc *HealthCheckImpl) getAliasByCell(cell string, topoServer *topo.Server) string { +func (hc *HealthCheckImpl) getAliasByCell(cell string) string { hc.mu.Lock() defer hc.mu.Unlock() @@ -1099,8 +1244,82 @@ func (hc *HealthCheckImpl) getAliasByCell(cell string, topoServer *topo.Server) return alias } - alias := topo.GetAliasByCell(context.Background(), topoServer, cell) + alias := topo.GetAliasByCell(context.Background(), hc.ts, cell) hc.cellAliases[cell] = alias return alias } + +func (hc *HealthCheckImpl) isTabletInCell(tablet *topodatapb.Tablet) bool { + if tablet.Type == topodatapb.TabletType_MASTER { + return true + } + if tablet.Alias.Cell == hc.cell { + return true + } + if hc.getAliasByCell(tablet.Alias.Cell) == hc.getAliasByCell(hc.cell) { + return true + } + return false +} + +// WaitForTablets waits for at least one tablet in the given +// keyspace / shard / tablet type before returning. The tablets do not +// have to be healthy. It will return ctx.Err() if the context is canceled. +func (hc *HealthCheckImpl) WaitForTablets(ctx context.Context, keyspace, shard string, tabletType topodatapb.TabletType) error { + targets := []*querypb.Target{ + { + Keyspace: keyspace, + Shard: shard, + TabletType: tabletType, + }, + } + return hc.waitForTablets(ctx, targets, false) +} + +// WaitForAllServingTablets waits for at least one healthy serving tablet in +// each given target before returning. +// It will return ctx.Err() if the context is canceled. +// It will return an error if it can't read the necessary topology records. +func (hc *HealthCheckImpl) WaitForAllServingTablets(ctx context.Context, targets []*querypb.Target) error { + return hc.waitForTablets(ctx, targets, true) +} + +// waitForTablets is the internal method that polls for tablets. +func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*querypb.Target, requireServing bool) error { + for { + // We nil targets as we find them. + allPresent := true + for i, target := range targets { + if target == nil { + continue + } + + var stats []tabletStats + if requireServing { + stats = hc.getHealthyTabletStats(target.Keyspace, target.Shard, target.TabletType) + } else { + stats = hc.getTabletStats(target.Keyspace, target.Shard, target.TabletType) + } + if len(stats) == 0 { + allPresent = false + } else { + targets[i] = nil + } + } + + if allPresent { + // we found everything we needed + return nil + } + + // Unblock after the sleep or when the context has expired. + timer := time.NewTimer(waitAvailableTabletInterval) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go new file mode 100644 index 00000000000..7e5d0af7d7e --- /dev/null +++ b/go/vt/discovery/healthcheck_test.go @@ -0,0 +1,206 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "bytes" + "flag" + "fmt" + "html/template" + "strings" + "testing" + "time" + + "vitess.io/vitess/go/vt/topo/memorytopo" + + "golang.org/x/net/context" + "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/status" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/vttablet/queryservice" + "vitess.io/vitess/go/vt/vttablet/tabletconn" + + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +func init() { + tabletconn.RegisterDialer("fake_gateway", tabletDialer) + flag.Set("tablet_protocol", "fake_gateway") +} + +func TestHealthCheck(t *testing.T) { + ts := memorytopo.NewServer("cell") + tablet := topo.NewTablet(0, "cell", "a") + tablet.PortMap["vt"] = 1 + input := make(chan *querypb.StreamHealthResponse) + createFakeConn(tablet, input) + t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) + hc := createTestHc(ts) + testChecksum(t, 0, hc.stateChecksum()) + hc.AddTablet(tablet, "") + t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + + // close healthcheck + hc.Close() +} + +func TestHealthCheckStreamError(t *testing.T) { + ts := memorytopo.NewServer("cell") + tablet := topo.NewTablet(0, "cell", "a") + tablet.PortMap["vt"] = 1 + input := make(chan *querypb.StreamHealthResponse) + fc := createFakeConn(tablet, input) + fc.errCh = make(chan error) + t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) + hc := createTestHc(ts) + hc.AddTablet(tablet, "") + t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + + // close healthcheck + hc.Close() +} + +func TestHealthCheckVerifiesTabletAlias(t *testing.T) { + ts := memorytopo.NewServer("cell") + t.Logf("starting") + tablet := topo.NewTablet(1, "cell", "a") + tablet.PortMap["vt"] = 1 + input := make(chan *querypb.StreamHealthResponse, 1) + createFakeConn(tablet, input) + + t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) + + hc := createTestHc(ts) + hc.AddTablet(tablet, "") + t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + + // close healthcheck + hc.Close() +} + +// TestHealthCheckCloseWaitsForGoRoutines tests that Close() waits for all Go +// routines to finish and the listener won't be called anymore. +func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { + ts := memorytopo.NewServer("cell") + tablet := topo.NewTablet(0, "cell", "a") + tablet.PortMap["vt"] = 1 + input := make(chan *querypb.StreamHealthResponse, 1) + createFakeConn(tablet, input) + + t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) + + hc := createTestHc(ts) + hc.AddTablet(tablet, "") + t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + + hc.Close() +} + +func TestHealthCheckTimeout(t *testing.T) { + ts := memorytopo.NewServer("cell") + timeout := 500 * time.Millisecond + tablet := topo.NewTablet(0, "cell", "a") + tablet.PortMap["vt"] = 1 + input := make(chan *querypb.StreamHealthResponse) + createFakeConn(tablet, input) + t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) + hc := createTestHc(ts) + hc.retryDelay = timeout + hc.AddTablet(tablet, "") + t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + + // close healthcheck + hc.Close() +} + +func TestTemplate(t *testing.T) { + tablet := topo.NewTablet(0, "cell", "a") + ts := []*tabletStats{ + { + Key: "a", + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Up: true, + Serving: false, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3}, + MasterTermStartTime: 0, + }, + } + tcs := &TabletsCacheStatus{ + Cell: "cell", + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + TabletsStats: ts, + } + templ := template.New("").Funcs(status.StatusFuncs) + templ, err := templ.Parse(HealthCheckTemplate) + if err != nil { + t.Fatalf("error parsing template: %v", err) + } + wr := &bytes.Buffer{} + if err := templ.Execute(wr, []*TabletsCacheStatus{tcs}); err != nil { + t.Fatalf("error executing template: %v", err) + } +} + +func TestDebugURLFormatting(t *testing.T) { + flag.Set("tablet_url_template", "https://{{.GetHostNameLevel 0}}.bastion.{{.Tablet.Alias.Cell}}.corp") + ParseTabletURLTemplateFromFlag() + + tablet := topo.NewTablet(0, "cell", "host.dc.domain") + ts := []*tabletStats{ + { + Key: "a", + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Up: true, + Serving: false, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3}, + MasterTermStartTime: 0, + }, + } + tcs := &TabletsCacheStatus{ + Cell: "cell", + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + TabletsStats: ts, + } + templ := template.New("").Funcs(status.StatusFuncs) + templ, err := templ.Parse(HealthCheckTemplate) + if err != nil { + t.Fatalf("error parsing template: %v", err) + } + wr := &bytes.Buffer{} + if err := templ.Execute(wr, []*TabletsCacheStatus{tcs}); err != nil { + t.Fatalf("error executing template: %v", err) + } + expectedURL := `"https://host.bastion.cell.corp"` + if !strings.Contains(wr.String(), expectedURL) { + t.Fatalf("output missing formatted URL, expectedURL: %s , output: %s", expectedURL, wr.String()) + } +} + +func tabletDialer(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) { + key := TabletToMapKey(tablet) + if qs, ok := connMap[key]; ok { + return qs, nil + } + return nil, fmt.Errorf("tablet %v not found", key) +} + +func createTestHc(ts *topo.Server) *HealthCheckImpl { + return NewHealthCheck(context.Background(), 1*time.Millisecond, time.Hour, ts, "cell").(*HealthCheckImpl) +} diff --git a/go/vt/discovery/legacy_healthcheck_flaky_test.go b/go/vt/discovery/legacy_healthcheck_flaky_test.go index dab8f5bb901..f77eca845ea 100644 --- a/go/vt/discovery/legacy_healthcheck_flaky_test.go +++ b/go/vt/discovery/legacy_healthcheck_flaky_test.go @@ -56,7 +56,7 @@ func testChecksum(t *testing.T, want, got int64) { } } -func TestHealthCheck(t *testing.T) { +func TestLegacyHealthCheck(t *testing.T) { tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 input := make(chan *querypb.StreamHealthResponse) @@ -243,7 +243,7 @@ func TestHealthCheck(t *testing.T) { hc.Close() } -func TestHealthCheckStreamError(t *testing.T) { +func TestLegacyHealthCheckStreamError(t *testing.T) { tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 input := make(chan *querypb.StreamHealthResponse) @@ -313,7 +313,7 @@ func TestHealthCheckStreamError(t *testing.T) { hc.Close() } -func TestHealthCheckVerifiesTabletAlias(t *testing.T) { +func TestLegacyHealthCheckVerifiesTabletAlias(t *testing.T) { t.Logf("starting") tablet := topo.NewTablet(1, "cell", "a") tablet.PortMap["vt"] = 1 @@ -378,9 +378,9 @@ func TestHealthCheckVerifiesTabletAlias(t *testing.T) { hc.Close() } -// TestHealthCheckCloseWaitsForGoRoutines tests that Close() waits for all Go +// TestLegacyHealthCheckCloseWaitsForGoRoutines tests that Close() waits for all Go // routines to finish and the listener won't be called anymore. -func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { +func TestLegacyHealthCheckCloseWaitsForGoRoutines(t *testing.T) { tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 input := make(chan *querypb.StreamHealthResponse, 1) @@ -476,7 +476,7 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { } } -func TestHealthCheckTimeout(t *testing.T) { +func TestLegacyHealthCheckTimeout(t *testing.T) { timeout := 500 * time.Millisecond tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 @@ -579,7 +579,7 @@ func TestHealthCheckTimeout(t *testing.T) { hc.Close() } -func TestTemplate(t *testing.T) { +func TestLegacyTemplate(t *testing.T) { tablet := topo.NewTablet(0, "cell", "a") ts := []*LegacyTabletStats{ { @@ -608,7 +608,7 @@ func TestTemplate(t *testing.T) { } } -func TestDebugURLFormatting(t *testing.T) { +func TestLegacyDebugURLFormatting(t *testing.T) { flag.Set("tablet_url_template", "https://{{.GetHostNameLevel 0}}.bastion.{{.Tablet.Alias.Cell}}.corp") ParseTabletURLTemplateFromFlag() diff --git a/go/vt/discovery/legacy_tablet_stats_cache_test.go b/go/vt/discovery/legacy_tablet_stats_cache_test.go index 97bf9eff4dd..4a50109cafe 100644 --- a/go/vt/discovery/legacy_tablet_stats_cache_test.go +++ b/go/vt/discovery/legacy_tablet_stats_cache_test.go @@ -28,7 +28,7 @@ import ( ) // TestTabletStatsCache tests the functionality of the LegacyTabletStatsCache class. -func TestTabletStatsCache(t *testing.T) { +func TestLegacyTabletStatsCache(t *testing.T) { ts := memorytopo.NewServer("cell", "cell1", "cell2") cellsAlias := &topodatapb.CellsAlias{ diff --git a/go/vt/discovery/tablet_stats_cache.go b/go/vt/discovery/tablet_stats_cache.go index 41bd2209b2f..0c4826c1cd6 100644 --- a/go/vt/discovery/tablet_stats_cache.go +++ b/go/vt/discovery/tablet_stats_cache.go @@ -19,8 +19,9 @@ package discovery import ( "sync" - "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" + + "vitess.io/vitess/go/vt/log" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" ) @@ -44,53 +45,6 @@ type tabletStatsCache struct { entries map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry } -// tabletStatsCacheEntry is the per keyspace/shard/tabletType -// entry of the in-memory map for tabletStatsCache. -type tabletStatsCacheEntry struct { - // mu protects the rest of this structure. - mu sync.Mutex - // all has the valid tablets, indexed by TabletToMapKey(ts.Tablet), - // as it is the index used by HealthCheck. - all map[string]*tabletStats - // healthy only has the healthy ones. - healthy []*tabletStats -} - -func (e *tabletStatsCacheEntry) updateHealthyMapForMaster(ts *tabletStats) { - if ts.Up { - // We have an Up master. - if len(e.healthy) == 0 { - // We have a new Up server, just remember it. - e.healthy = append(e.healthy, ts) - return - } - - // We already have one up server, see if we - // need to replace it. - if ts.MasterTermStartTime < e.healthy[0].MasterTermStartTime { - log.Warningf("not marking healthy master %s as Up for %s because its externally reparented timestamp is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", - topoproto.TabletAliasString(ts.Tablet.Alias), - topoproto.KeyspaceShardString(ts.Target.Keyspace, ts.Target.Shard), - topoproto.TabletAliasString(e.healthy[0].Tablet.Alias), - ts.MasterTermStartTime, - e.healthy[0].MasterTermStartTime) - return - } - - // Just replace it. - e.healthy[0] = ts - return - } - - // We have a Down master, remove it only if it's exactly the same. - if len(e.healthy) != 0 { - if ts.Key == e.healthy[0].Key { - // Same guy, remove it. - e.healthy = nil - } - } -} - func newTabletStatsCache() *tabletStatsCache { tc := &tabletStatsCache{ entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), @@ -146,6 +100,76 @@ func (tc *tabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStat return e } +// tabletStatsCacheEntry is the per keyspace/shard/tabletType +// entry of the in-memory map for tabletStatsCache. +type tabletStatsCacheEntry struct { + // mu protects the rest of this structure. + mu sync.Mutex + // all has the valid tablets, indexed by TabletToMapKey(ts.Tablet), + // as it is the index used by HealthCheck. + all map[string]*tabletStats + // healthy only has the healthy ones. + healthy []*tabletStats +} + +func (e *tabletStatsCacheEntry) updateHealthyMapForMaster(ts *tabletStats) { + if ts.Target.TabletType != topodatapb.TabletType_MASTER { + panic("program bug") + } + if ts.Up { + // We have an Up master. + if len(e.healthy) == 0 { + // We have a new Up server, just remember it. + e.healthy = append(e.healthy, ts) + return + } + + // We already have one up server, see if we + // need to replace it. + if ts.MasterTermStartTime < e.healthy[0].MasterTermStartTime { + log.Warningf("not marking healthy master %s as Up for %s because its externally reparented timestamp is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", + topoproto.TabletAliasString(ts.Tablet.Alias), + topoproto.KeyspaceShardString(ts.Target.Keyspace, ts.Target.Shard), + topoproto.TabletAliasString(e.healthy[0].Tablet.Alias), + ts.MasterTermStartTime, + e.healthy[0].MasterTermStartTime) + return + } + + // Just replace it. + e.healthy[0] = ts + return + } + + // We have a Down master, remove it only if it's exactly the same. + if len(e.healthy) != 0 { + if ts.Key == e.healthy[0].Key { + // Same guy, remove it. + e.healthy = nil + } + } +} + +func (e *tabletStatsCacheEntry) getHealthyTabletStats() []tabletStats { + e.mu.Lock() + defer e.mu.Unlock() + result := make([]tabletStats, len(e.healthy)) + for i, ts := range e.healthy { + result[i] = *ts + } + return result +} + +func (e *tabletStatsCacheEntry) getTabletStats() []tabletStats { + e.mu.Lock() + defer e.mu.Unlock() + result := make([]tabletStats, 0, len(e.all)) + for _, ts := range e.all { + result = append(result, *ts) + } + return result +} + // ResetForTesting is for use in tests only. func (tc *tabletStatsCache) ResetForTesting() { tc.mu.Lock() diff --git a/go/vt/discovery/tablet_stats_cache_test.go b/go/vt/discovery/tablet_stats_cache_test.go new file mode 100644 index 00000000000..84002dab5b9 --- /dev/null +++ b/go/vt/discovery/tablet_stats_cache_test.go @@ -0,0 +1,92 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo" +) + +// TestTabletStatsCache tests the functionality of the TabletStatsCache class. +func TestTabletStatsCache(t *testing.T) { + // We want to unit test TabletStatsCache without a full-blown + // HealthCheck object, so we can't call NewTabletStatsCache. + // So we just construct this object here. + tsc := &tabletStatsCache{ + entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), + } + + // empty + a := tsc.getEntry("k", "s", topodatapb.TabletType_MASTER) + assert.Nil(t, a) + // add a tablet + b := tsc.getOrCreateEntry(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.NotNil(t, b) + tablet1 := topo.NewTablet(10, "cell", "host1") + ts1 := &tabletStats{ + Key: "t1", + Tablet: tablet1, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Up: true, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + b.all[TabletToMapKey(tablet1)] = ts1 + + // check it's there + c := tsc.getOrCreateEntry(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.NotNil(t, c) + got := c.getTabletStats() + assert.Equal(t, 1, len(got)) + assert.True(t, ts1.DeepEqual(&got[0])) + + // add a second tablet + tablet2 := topo.NewTablet(11, "cell", "host2") + ts2 := &tabletStats{ + Key: "t2", + Tablet: tablet2, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, + Up: true, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 0, CpuUsage: 0.2}, + } + d := tsc.getOrCreateEntry(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) + d.all[TabletToMapKey(tablet2)] = ts2 + + d.updateHealthyMapForMaster(ts2) + // should be in healthy tablet stats + got = d.getHealthyTabletStats() + assert.Equal(t, 1, len(got)) + assert.True(t, ts2.DeepEqual(&got[0])) + // should be in all tabletStats + got = d.getTabletStats() + assert.Equal(t, 1, len(got)) + assert.True(t, ts2.DeepEqual(&got[0])) + + // master goes down + ts2.Up = false + d.updateHealthyMapForMaster(ts2) + got = d.getHealthyTabletStats() + // check it is not there + assert.Equal(t, 0, len(got)) + +} diff --git a/go/vt/discovery/topology_watcher.go b/go/vt/discovery/topology_watcher.go index 9648c6dfaae..c4c7adf9c79 100644 --- a/go/vt/discovery/topology_watcher.go +++ b/go/vt/discovery/topology_watcher.go @@ -17,10 +17,7 @@ limitations under the License. package discovery import ( - "bytes" "fmt" - "hash/crc32" - "sort" "strings" "sync" "time" @@ -32,10 +29,8 @@ import ( "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/topoproto" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo" ) const ( @@ -62,42 +57,19 @@ type tabletInfo struct { // NewCellTabletsWatcher returns a TopologyWatcher that monitors all // the tablets in a cell, and starts refreshing. -func NewCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *TopologyWatcher { - return NewTopologyWatcher(ctx, topoServer, tr, cell, refreshInterval, refreshKnownTablets, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) { +func NewCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, f TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *TopologyWatcher { + return NewTopologyWatcher(ctx, topoServer, f, cell, refreshInterval, refreshKnownTablets, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) { return tw.topoServer.GetTabletsByCell(ctx, tw.cell) }) } -// NewShardReplicationWatcher returns a TopologyWatcher that -// monitors the tablets in a cell/keyspace/shard, and starts refreshing. -func NewShardReplicationWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) *TopologyWatcher { - return NewTopologyWatcher(ctx, topoServer, tr, cell, refreshInterval, true /* RefreshKnownTablets */, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) { - sri, err := tw.topoServer.GetShardReplication(ctx, tw.cell, keyspace, shard) - switch { - case err == nil: - // we handle this case after this switch block - case topo.IsErrType(err, topo.NoNode): - // this is not an error - return nil, nil - default: - return nil, err - } - - result := make([]*topodatapb.TabletAlias, len(sri.Nodes)) - for i, node := range sri.Nodes { - result[i] = node.TabletAlias - } - return result, nil - }) -} - // TopologyWatcher polls tablet from a configurable set of tablets // periodically. When tablets are added / removed, it calls // the TabletRecorder AddTablet / RemoveTablet interface appropriately. type TopologyWatcher struct { // set at construction time topoServer *topo.Server - tr TabletRecorder + tabletFilter TabletFilter cell string refreshInterval time.Duration refreshKnownTablets bool @@ -124,10 +96,10 @@ type TopologyWatcher struct { // NewTopologyWatcher returns a TopologyWatcher that monitors all // the tablets in a cell, and starts refreshing. -func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error)) *TopologyWatcher { +func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, filter TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error)) *TopologyWatcher { tw := &TopologyWatcher{ topoServer: topoServer, - tr: tr, + tabletFilter: filter, cell: cell, refreshInterval: refreshInterval, refreshKnownTablets: refreshKnownTablets, @@ -140,152 +112,9 @@ func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr TabletR // We want the span from the context, but not the cancelation that comes with it spanContext := trace.CopySpan(context.Background(), ctx) tw.ctx, tw.cancelFunc = context.WithCancel(spanContext) - tw.wg.Add(1) - go tw.watch() return tw } -// watch polls all tablets and notifies TabletRecorder by adding/removing tablets. -func (tw *TopologyWatcher) watch() { - defer tw.wg.Done() - ticker := time.NewTicker(tw.refreshInterval) - defer ticker.Stop() - for { - tw.loadTablets() - select { - case <-tw.ctx.Done(): - return - case <-ticker.C: - } - } -} - -// loadTablets reads all tablets from topology, and updates TabletRecorder. -func (tw *TopologyWatcher) loadTablets() { - var wg sync.WaitGroup - newTablets := make(map[string]*tabletInfo) - replacedTablets := make(map[string]*tabletInfo) - - tabletAliases, err := tw.getTablets(tw) - topologyWatcherOperations.Add(topologyWatcherOpListTablets, 1) - if err != nil { - topologyWatcherErrors.Add(topologyWatcherOpListTablets, 1) - select { - case <-tw.ctx.Done(): - return - default: - } - log.Errorf("cannot get tablets for cell: %v: %v", tw.cell, err) - return - } - - // Accumulate a list of all known alias strings to use later - // when sorting - tabletAliasStrs := make([]string, 0, len(tabletAliases)) - - tw.mu.Lock() - for _, tAlias := range tabletAliases { - aliasStr := topoproto.TabletAliasString(tAlias) - tabletAliasStrs = append(tabletAliasStrs, aliasStr) - - if !tw.refreshKnownTablets { - if val, ok := tw.tablets[aliasStr]; ok { - newTablets[aliasStr] = val - continue - } - } - - wg.Add(1) - go func(alias *topodatapb.TabletAlias) { - defer wg.Done() - tw.sem <- 1 // Wait for active queue to drain. - tablet, err := tw.topoServer.GetTablet(tw.ctx, alias) - topologyWatcherOperations.Add(topologyWatcherOpGetTablet, 1) - <-tw.sem // Done; enable next request to run - if err != nil { - topologyWatcherErrors.Add(topologyWatcherOpGetTablet, 1) - select { - case <-tw.ctx.Done(): - return - default: - } - log.Errorf("cannot get tablet for alias %v: %v", alias, err) - return - } - tw.mu.Lock() - aliasStr := topoproto.TabletAliasString(alias) - newTablets[aliasStr] = &tabletInfo{ - alias: aliasStr, - key: TabletToMapKey(tablet.Tablet), - tablet: tablet.Tablet, - } - tw.mu.Unlock() - }(tAlias) - } - - tw.mu.Unlock() - wg.Wait() - tw.mu.Lock() - - for alias, newVal := range newTablets { - if val, ok := tw.tablets[alias]; !ok { - // Check if there's a tablet with the same address key but a - // different alias. If so, replace it and keep track of the - // replaced alias to make sure it isn't removed later. - found := false - for _, otherVal := range tw.tablets { - if newVal.key == otherVal.key { - found = true - tw.tr.ReplaceTablet(otherVal.tablet, newVal.tablet, alias) - topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) - replacedTablets[otherVal.alias] = newVal - } - } - if !found { - tw.tr.AddTablet(newVal.tablet, alias) - topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1) - } - - } else if val.key != newVal.key { - // Handle the case where the same tablet alias is now reporting - // a different address key. - replacedTablets[alias] = newVal - tw.tr.ReplaceTablet(val.tablet, newVal.tablet, alias) - topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) - } - } - - for _, val := range tw.tablets { - if _, ok := newTablets[val.alias]; !ok { - if _, ok2 := replacedTablets[val.alias]; !ok2 { - tw.tr.RemoveTablet(val.tablet) - topologyWatcherOperations.Add(topologyWatcherOpRemoveTablet, 1) - } - } - } - tw.tablets = newTablets - if !tw.firstLoadDone { - tw.firstLoadDone = true - close(tw.firstLoadChan) - } - - // iterate through the tablets in a stable order and compute a - // checksum of the tablet map - sort.Strings(tabletAliasStrs) - var buf bytes.Buffer - for _, alias := range tabletAliasStrs { - tabletInfo, ok := tw.tablets[alias] - if ok { - buf.WriteString(alias) - buf.WriteString(tabletInfo.key) - } - } - tw.topoChecksum = crc32.ChecksumIEEE(buf.Bytes()) - tw.lastRefresh = time.Now() - - tw.mu.Unlock() -} - // WaitForInitialTopology waits until the watcher reads all of the topology data // for the first time and transfers the information to TabletRecorder via its // AddTablet() method. @@ -321,10 +150,16 @@ func (tw *TopologyWatcher) TopoChecksum() uint32 { return tw.topoChecksum } +// TabletFilter is an interface that can be given to a TopologyWatcher +// to be applied as an additional filter on the list of tablets returned by its getTablets function +type TabletFilter interface { + // IsIncluded returns whether tablet is included in this filter + IsIncluded(tablet *topodatapb.Tablet) bool +} + // FilterByShard is a filter that filters tablets by // keyspace/shard. type FilterByShard struct { - cell string // filters is a map of keyspace to filters for shards filters map[string][]*filterShard } @@ -342,7 +177,7 @@ type filterShard struct { // can either be a shard name, or a keyrange. All tablets that match // at least one keyspace|shard tuple will be forwarded to the // underlying TabletRecorder. -func NewFilterByShard(cell string, filters []string) (*FilterByShard, error) { +func NewFilterByShard(filters []string) (*FilterByShard, error) { m := make(map[string][]*filterShard) for _, filter := range filters { parts := strings.Split(filter, "|") @@ -374,7 +209,6 @@ func NewFilterByShard(cell string, filters []string) (*FilterByShard, error) { } return &FilterByShard{ - cell: cell, filters: m, }, nil } @@ -382,10 +216,6 @@ func NewFilterByShard(cell string, filters []string) (*FilterByShard, error) { // IsIncluded returns true iff the tablet's keyspace and shard should be // forwarded to the underlying TabletRecorder. func (fbs *FilterByShard) IsIncluded(tablet *topodatapb.Tablet) bool { - if !isTabletInCell(fbs.cell, tablet) { - return false - } - canonical, kr, err := topo.ValidateShardName(tablet.Shard) if err != nil { log.Errorf("Error parsing shard name %v, will ignore tablet: %v", tablet.Shard, err) @@ -408,7 +238,6 @@ func (fbs *FilterByShard) IsIncluded(tablet *topodatapb.Tablet) bool { // FilterByKeyspace is a filter that filters tablets by // keyspace type FilterByKeyspace struct { - cell string keyspaces map[string]bool } @@ -422,7 +251,6 @@ func NewFilterByKeyspace(cell string, selectedKeyspaces []string) *FilterByKeysp } return &FilterByKeyspace{ - cell: cell, keyspaces: m, } } @@ -430,23 +258,6 @@ func NewFilterByKeyspace(cell string, selectedKeyspaces []string) *FilterByKeysp // IsIncluded returns true if the tablet's keyspace should be // forwarded to the underlying TabletRecorder. func (fbk *FilterByKeyspace) IsIncluded(tablet *topodatapb.Tablet) bool { - if !isTabletInCell(fbk.cell, tablet) { - return false - } _, exist := fbk.keyspaces[tablet.Keyspace] return exist } - -func isTabletInCell(cell string, tablet *topodatapb.Tablet) bool { - if tablet.Type == topodatapb.TabletType_MASTER { - return true - } - if tablet.Alias.Cell == cell { - return true - } - // TODO(deepthi): need to implement cell aliases here otherwise they won't work - //if getAliasByCell(tablet.Alias.Cell) == getAliasByCell(cell) { - // return true - //} - return false -} diff --git a/go/vt/vtgate/discoverygateway.go b/go/vt/vtgate/discoverygateway.go index ff0ca4c30ac..3148a65bf89 100644 --- a/go/vt/vtgate/discoverygateway.go +++ b/go/vt/vtgate/discoverygateway.go @@ -107,9 +107,9 @@ func NewDiscoveryGateway(ctx context.Context, hc discovery.LegacyHealthCheck, se // We set sendDownEvents=true because it's required by LegacyTabletStatsCache. hc.SetListener(dg, true /* sendDownEvents */) - cellsToWatch := *discovery.CellsToWatch - log.Infof("loading tablets for cells: %v", cellsToWatch) - for _, c := range strings.Split(cellsToWatch, ",") { + cells := *discovery.CellsToWatch + log.Infof("loading tablets for cells: %v", cells) + for _, c := range strings.Split(cells, ",") { if c == "" { continue } diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index 33055032674..f2eee9a5bf3 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -79,7 +79,6 @@ func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string log.Exitf("Unable to create new TabletGateway: %v", err) } } - hc := discovery.NewHealthCheck(ctx, *HealthCheckRetryDelay, *HealthCheckTimeout, topoServer, localCell) gw := &TabletGateway{ @@ -90,10 +89,8 @@ func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string statusAggregators: make(map[string]*TabletStatusAggregator), buffer: buffer.New(), } - - // TODO(deepthi): start healthcheck here - //hc.Open() - + // Start the healthcheck + hc.Open() gw.QueryService = queryservice.Wrap(nil, gw.withRetry) return gw } @@ -121,13 +118,11 @@ func (gw *TabletGateway) WaitForTablets(ctx context.Context, tabletTypesToWait [ } // Finds the targets to look for. - _, err := srvtopo.FindAllTargets(ctx, gw.srvTopoServer, gw.localCell, tabletTypesToWait) + targets, err := srvtopo.FindAllTargets(ctx, gw.srvTopoServer, gw.localCell, tabletTypesToWait) if err != nil { return err } - return nil - // TODO(deepthi): this needs to be implemented - //return gw.hc.WaitForAllServingTablets(ctx, targets) + return gw.hc.WaitForAllServingTablets(ctx, targets) } // Close shuts down underlying connections. From d4cb21467dc0a2095cd8bc636d101cb56bc38ebb Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 23 Apr 2020 14:13:58 -0700 Subject: [PATCH 06/39] healthcheck: remove unnecessary flag, fix some TODOs Signed-off-by: deepthi --- go/cmd/vtgate/status.go | 2 +- go/cmd/vtgate/vtgate.go | 6 +++--- go/vt/discovery/healthcheck.go | 1 - go/vt/discovery/tablet_stats.go | 2 -- go/vt/vtgate/discoverygateway.go | 5 +++-- go/vt/vtgate/gateway.go | 9 +++++---- go/vt/vtgate/scatter_conn.go | 2 +- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/go/cmd/vtgate/status.go b/go/cmd/vtgate/status.go index d714e444b90..0ae2f18b789 100644 --- a/go/cmd/vtgate/status.go +++ b/go/cmd/vtgate/status.go @@ -38,7 +38,7 @@ func addStatusParts(vtg *vtgate.VTGate) { servenv.AddStatusPart("Gateway Status", vtgate.StatusTemplate, func() interface{} { return vtg.GetGatewayCacheStatus() }) - if *useLegacyHealthCheck { + if *vtgate.GatewayImplementation == vtgate.GatewayImplementationDiscovery { servenv.AddStatusPart("Health Check Cache", discovery.HealthCheckTemplate, func() interface{} { return legacyHealthCheck.CacheStatus() }) diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index cbe0e3de23d..9de94d31c0f 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -39,8 +39,6 @@ import ( var ( cell = flag.String("cell", "test_nj", "cell to use") tabletTypesToWait = flag.String("tablet_types_to_wait", "", "wait till connected for specified tablet types during Gateway initialization") - //TODO(deepthi): remove this and use gateway implementation as the flag. discovery => true, tablet => false - useLegacyHealthCheck = flag.Bool("use_legacy_health_check", true, "whether to use the legacy health check") ) var resilientServer *srvtopo.ResilientServer @@ -76,12 +74,14 @@ func main() { } var vtg *vtgate.VTGate - if *useLegacyHealthCheck { + if *vtgate.GatewayImplementation == vtgate.GatewayImplementationDiscovery { + // default value legacyHealthCheck = discovery.NewLegacyHealthCheck(*vtgate.HealthCheckRetryDelay, *vtgate.HealthCheckTimeout) legacyHealthCheck.RegisterStats() vtg = vtgate.LegacyInit(context.Background(), legacyHealthCheck, resilientServer, *cell, *vtgate.RetryCount, tabletTypes) } else { + // use new Init otherwise vtg = vtgate.Init(context.Background(), resilientServer, *cell, tabletTypes) } diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index ea63f8258ed..9551d8ad3a4 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -862,7 +862,6 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St } hcc.setServingState(serving, reason) - // TODO(deepthi): do we need a copy? updateHealth should be done by HealthCheckConn, not healthCheck hc.updateHealth(hcc.tabletStats.Copy(), hcc.conn) return nil } diff --git a/go/vt/discovery/tablet_stats.go b/go/vt/discovery/tablet_stats.go index fc72577271f..3d623426a9a 100644 --- a/go/vt/discovery/tablet_stats.go +++ b/go/vt/discovery/tablet_stats.go @@ -44,8 +44,6 @@ type tabletStats struct { // LastError is the error we last saw when trying to get the // tablet's healthcheck. LastError error - // TODO(deepthi): No member of this struct should be accessed without holding the mutex - // mu sync.Mutex } // String is defined because we want to print a []*tabletStats array nicely. diff --git a/go/vt/vtgate/discoverygateway.go b/go/vt/vtgate/discoverygateway.go index 3148a65bf89..b78eb710cbf 100644 --- a/go/vt/vtgate/discoverygateway.go +++ b/go/vt/vtgate/discoverygateway.go @@ -43,11 +43,12 @@ import ( ) const ( - gatewayImplementationDiscovery = "discoverygateway" + // GatewayImplementationDiscovery defines the string value used as the implementation key for DiscoveryGateway + GatewayImplementationDiscovery = "discoverygateway" ) func init() { - RegisterGatewayCreator(gatewayImplementationDiscovery, createDiscoveryGateway) + RegisterGatewayCreator(GatewayImplementationDiscovery, createDiscoveryGateway) } // DiscoveryGateway is the default Gateway implementation. diff --git a/go/vt/vtgate/gateway.go b/go/vt/vtgate/gateway.go index a6b32d14cff..1eb65722db9 100644 --- a/go/vt/vtgate/gateway.go +++ b/go/vt/vtgate/gateway.go @@ -33,8 +33,9 @@ import ( // a query targeted to a keyspace/shard/tablet_type and send it off. var ( - implementation = flag.String("gateway_implementation", "discoverygateway", "The implementation of gateway") - initialTabletTimeout = flag.Duration("gateway_initial_tablet_timeout", 30*time.Second, "At startup, the gateway will wait up to that duration to get one tablet per keyspace/shard/tablettype") + // GatewayImplementation allows you to choose which gateway to use for vtgate routing. Defaults to discoverygateway, other option is tabletgateway + GatewayImplementation = flag.String("gateway_implementation", "discoverygateway", "Allowed values: discoverygateway (default), tabletgateway") + initialTabletTimeout = flag.Duration("gateway_initial_tablet_timeout", 30*time.Second, "At startup, the gateway will wait up to that duration to get one tablet per keyspace/shard/tablettype") // RetryCount is the number of times a query will be retried on error // Make this unexported after DiscoveryGateway is deprecated RetryCount = flag.Int("retry-count", 2, "retry count") @@ -78,9 +79,9 @@ func RegisterGatewayCreator(name string, gc Creator) { // GatewayCreator returns the Creator specified by the gateway_implementation flag. func GatewayCreator() Creator { - gc, ok := creators[*implementation] + gc, ok := creators[*GatewayImplementation] if !ok { - log.Exitf("No gateway registered as %s", *implementation) + log.Exitf("No gateway registered as %s", *GatewayImplementation) } return gc } diff --git a/go/vt/vtgate/scatter_conn.go b/go/vt/vtgate/scatter_conn.go index 4755d58451e..6e2095acb0a 100644 --- a/go/vt/vtgate/scatter_conn.go +++ b/go/vt/vtgate/scatter_conn.go @@ -107,7 +107,7 @@ func NewScatterConn(statsName string, txConn *TxConn, gw *TabletGateway) *Scatte []string{"Operation", "Keyspace", "ShardName", "DbType"}), txConn: txConn, gateway: gw, - //TODO(deepthi): we need to get ScatterConn working without using legacyHealthCheck + // gateway has a reference to healthCheck so we don't need this any more legacyHealthCheck: nil, } } From 21b9f74e835c50356238c45ad9a6acc0c94ccea3 Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 27 Apr 2020 21:01:11 -0700 Subject: [PATCH 07/39] healthcheck: simplify healthcheck struct by deleting tablet_stats_cache. rename tabletStats to tabletHealth Signed-off-by: deepthi --- go/cmd/vtcombo/status.go | 2 +- go/cmd/vtgate/status.go | 2 +- go/vt/discovery/healthcheck.go | 481 +++++++----------- go/vt/discovery/healthcheck_test.go | 16 +- go/vt/discovery/legacy_healthcheck.go | 63 +++ .../legacy_healthcheck_flaky_test.go | 4 +- go/vt/discovery/replicationlag.go | 30 +- go/vt/discovery/replicationlag_test.go | 46 +- .../{tablet_stats.go => tablet_health.go} | 96 ++-- go/vt/discovery/tablet_stats_cache.go | 179 ------- go/vt/discovery/tablet_stats_cache_test.go | 92 ---- go/vt/discovery/tablets_cache_status.go | 87 ++++ go/vt/discovery/utils.go | 20 - go/vt/vtgate/api.go | 4 +- go/vt/vtgate/tabletgateway.go | 2 - 15 files changed, 429 insertions(+), 695 deletions(-) rename go/vt/discovery/{tablet_stats.go => tablet_health.go} (56%) delete mode 100644 go/vt/discovery/tablet_stats_cache.go delete mode 100644 go/vt/discovery/tablet_stats_cache_test.go create mode 100644 go/vt/discovery/tablets_cache_status.go diff --git a/go/cmd/vtcombo/status.go b/go/cmd/vtcombo/status.go index 4d9b3079a94..c7524cb17d4 100644 --- a/go/cmd/vtcombo/status.go +++ b/go/cmd/vtcombo/status.go @@ -43,7 +43,7 @@ func addStatusParts(vtg *vtgate.VTGate) { servenv.AddStatusPart("Gateway Status", vtgate.StatusTemplate, func() interface{} { return vtg.GetGatewayCacheStatus() }) - servenv.AddStatusPart("Health Check Cache", discovery.HealthCheckTemplate, func() interface{} { + servenv.AddStatusPart("Health Check Cache", discovery.LegacyHealthCheckTemplate, func() interface{} { return healthCheck.CacheStatus() }) } diff --git a/go/cmd/vtgate/status.go b/go/cmd/vtgate/status.go index 0ae2f18b789..564f1b839df 100644 --- a/go/cmd/vtgate/status.go +++ b/go/cmd/vtgate/status.go @@ -39,7 +39,7 @@ func addStatusParts(vtg *vtgate.VTGate) { return vtg.GetGatewayCacheStatus() }) if *vtgate.GatewayImplementation == vtgate.GatewayImplementationDiscovery { - servenv.AddStatusPart("Health Check Cache", discovery.HealthCheckTemplate, func() interface{} { + servenv.AddStatusPart("Health Check Cache", discovery.LegacyHealthCheckTemplate, func() interface{} { return legacyHealthCheck.CacheStatus() }) } else { diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 9551d8ad3a4..be37dea588f 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -36,6 +36,7 @@ package discovery import ( "bytes" + "context" "encoding/json" "flag" "fmt" @@ -56,7 +57,6 @@ import ( "vitess.io/vitess/go/vt/topo" "github.com/golang/protobuf/proto" - "golang.org/x/net/context" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/sync2" "vitess.io/vitess/go/vt/grpcclient" @@ -99,17 +99,16 @@ var ( TopoReadConcurrency = flag.Int("topo_read_concurrency", 32, "concurrent topo reads") ) -// See the documentation for NewLegacyHealthCheck below for an explanation of these parameters. +// See the documentation for NewHealthCheck below for an explanation of these parameters. const ( DefaultHealthCheckRetryDelay = 5 * time.Second DefaultHealthCheckTimeout = 1 * time.Minute - // DefaultTopoReadConcurrency can be used as default value for the TopoReadConcurrency parameter of a LegacyTopologyWatcher. + // DefaultTopoReadConcurrency can be used as default value for the TopoReadConcurrency parameter of a TopologyWatcher. DefaultTopoReadConcurrency int = 5 // DefaultTopologyWatcherRefreshInterval can be used as the default value for // the refresh interval of a topology watcher. DefaultTopologyWatcherRefreshInterval = 1 * time.Minute - // HealthCheckTemplate is the HTML code to display a TabletsCacheStatusList HealthCheckTemplate = ` + + + + + + + + + + + + {{range $i, $ts := .}} + + + + + + + + {{end}} +
HealthCheck Tablet Cache
CellKeyspaceShardTabletTypetabletStats
{{github_com_vitessio_vitess_vtctld_srv_cell $ts.Cell}}{{github_com_vitessio_vitess_vtctld_srv_keyspace $ts.Cell $ts.Target.Keyspace}}{{$ts.Target.Shard}}{{$ts.Target.TabletType}}{{$ts.StatusAsHTML}}
+` +) + func init() { // Flags are not parsed at this point and the default value of the flag (just the hostname) will be used. ParseTabletURLTemplateFromFlag() @@ -215,6 +251,21 @@ func (e *LegacyTabletStats) TrivialStatsUpdate(n *LegacyTabletStats) bool { return false } +// TabletRecorder is the part of the LegacyHealthCheck interface that can +// add or remove tablets. We define it as a sub-interface here so we +// can add filters on tablets if needed. +type TabletRecorder interface { + // AddTablet adds the tablet. + // Name is an alternate name, like an address. + AddTablet(tablet *topodatapb.Tablet, name string) + + // RemoveTablet removes the tablet. + RemoveTablet(tablet *topodatapb.Tablet) + + // ReplaceTablet does an AddTablet and RemoveTablet in one call, effectively replacing the old tablet with the new. + ReplaceTablet(old, new *topodatapb.Tablet, name string) +} + // LegacyHealthCheck defines the interface of health checking module. // The goal of this object is to maintain a StreamHealth RPC // to a lot of tablets. Tablets are added / removed by calling the @@ -902,3 +953,15 @@ func (hc *LegacyHealthCheckImpl) Close() error { return nil } + +// TabletToMapKey creates a key to the map from tablet's host and ports. +// It should only be used in discovery and related module. +func TabletToMapKey(tablet *topodatapb.Tablet) string { + parts := make([]string, 0, 1) + for name, port := range tablet.PortMap { + parts = append(parts, netutil.JoinHostPort(name, port)) + } + sort.Strings(parts) + parts = append([]string{tablet.Hostname}, parts...) + return strings.Join(parts, ",") +} diff --git a/go/vt/discovery/legacy_healthcheck_flaky_test.go b/go/vt/discovery/legacy_healthcheck_flaky_test.go index f77eca845ea..60f8a1f7c10 100644 --- a/go/vt/discovery/legacy_healthcheck_flaky_test.go +++ b/go/vt/discovery/legacy_healthcheck_flaky_test.go @@ -598,7 +598,7 @@ func TestLegacyTemplate(t *testing.T) { TabletsStats: ts, } templ := template.New("").Funcs(status.StatusFuncs) - templ, err := templ.Parse(HealthCheckTemplate) + templ, err := templ.Parse(LegacyHealthCheckTemplate) if err != nil { t.Fatalf("error parsing template: %v", err) } @@ -630,7 +630,7 @@ func TestLegacyDebugURLFormatting(t *testing.T) { TabletsStats: ts, } templ := template.New("").Funcs(status.StatusFuncs) - templ, err := templ.Parse(HealthCheckTemplate) + templ, err := templ.Parse(LegacyHealthCheckTemplate) if err != nil { t.Fatalf("error parsing template: %v", err) } diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index ec4ffaa01ce..c1257d584d5 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -29,20 +29,20 @@ var ( minNumTablets = flag.Int("min_number_serving_vttablets", 2, "the minimum number of vttablets that will be continue to be used even with low replication lag") ) -// IsReplicationLagHigh verifies that the given LegacyTabletStats refers to a tablet with high +// IsReplicationLagHigh verifies that the given LegacytabletHealth refers to a tablet with high // replication lag, i.e. higher than the configured discovery_low_replication_lag flag. -func IsReplicationLagHigh(tabletStats *tabletStats) bool { - return float64(tabletStats.Stats.SecondsBehindMaster) > lowReplicationLag.Seconds() +func IsReplicationLagHigh(tabletHealth *tabletHealth) bool { + return float64(tabletHealth.Stats.SecondsBehindMaster) > lowReplicationLag.Seconds() } -// IsReplicationLagVeryHigh verifies that the given LegacyTabletStats refers to a tablet with very high +// IsReplicationLagVeryHigh verifies that the given LegacytabletHealth refers to a tablet with very high // replication lag, i.e. higher than the configured discovery_high_replication_lag_minimum_serving flag. -func IsReplicationLagVeryHigh(tabletStats *tabletStats) bool { - return float64(tabletStats.Stats.SecondsBehindMaster) > highReplicationLagMinServing.Seconds() +func IsReplicationLagVeryHigh(tabletHealth *tabletHealth) bool { + return float64(tabletHealth.Stats.SecondsBehindMaster) > highReplicationLagMinServing.Seconds() } -// FilterStatsByReplicationLag filters the list of LegacyTabletStats by LegacyTabletStats.Stats.SecondsBehindMaster. -// Note that LegacyTabletStats that is non-serving or has error is ignored. +// FilterStatsByReplicationLag filters the list of tabletHealth by tabletHealth.Stats.SecondsBehindMaster. +// Note that tabletHealth that is non-serving or has error is ignored. // // The simplified logic: // - Return tablets that have lag <= lowReplicationLag. @@ -66,14 +66,14 @@ func IsReplicationLagVeryHigh(tabletStats *tabletStats) bool { // The default for this is 2h, same as the discovery_high_replication_lag_minimum_serving here. // * degraded_threshold: this is only used by vttablet for display. It should match // discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. -func FilterStatsByReplicationLag(tabletStatsList []*tabletStats) []*tabletStats { - return filterStatsByLag(tabletStatsList) +func FilterStatsByReplicationLag(tabletHealthList []*tabletHealth) []*tabletHealth { + return filterStatsByLag(tabletHealthList) } -func filterStatsByLag(tabletStatsList []*tabletStats) []*tabletStats { - list := make([]tabletLagSnapshot, 0, len(tabletStatsList)) +func filterStatsByLag(tabletHealthList []*tabletHealth) []*tabletHealth { + list := make([]tabletLagSnapshot, 0, len(tabletHealthList)) // filter non-serving tablets and those with very high replication lag - for _, ts := range tabletStatsList { + for _, ts := range tabletHealthList { if !ts.Serving || ts.LastError != nil || ts.Stats == nil || IsReplicationLagVeryHigh(ts) { continue } @@ -87,7 +87,7 @@ func filterStatsByLag(tabletStatsList []*tabletStats) []*tabletStats { sort.Sort(tabletLagSnapshotList(list)) // Pick those with low replication lag, but at least minNumTablets tablets regardless. - res := make([]*tabletStats, 0, len(list)) + res := make([]*tabletHealth, 0, len(list)) for i := 0; i < len(list); i++ { if !IsReplicationLagHigh(list[i].ts) || i < *minNumTablets { res = append(res, list[i].ts) @@ -97,7 +97,7 @@ func filterStatsByLag(tabletStatsList []*tabletStats) []*tabletStats { } type tabletLagSnapshot struct { - ts *tabletStats + ts *tabletHealth replag uint32 } type tabletLagSnapshotList []tabletLagSnapshot diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index 57a64ef521d..79aedc30d7c 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -31,17 +31,17 @@ func testSetMinNumTablets(newMin int) { func TestFilterByReplicationLagUnhealthy(t *testing.T) { // 1 healthy serving tablet, 1 not healhty - ts1 := &tabletStats{ + ts1 := &tabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{}, } - ts2 := &tabletStats{ + ts2 := &tabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: false, Stats: &querypb.RealtimeStats{}, } - got := FilterStatsByReplicationLag([]*tabletStats{ts1, ts2}) + got := FilterStatsByReplicationLag([]*tabletHealth{ts1, ts2}) if len(got) != 1 { t.Errorf("len(FilterStatsByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}])) = %v, want 1", len(got)) } @@ -102,9 +102,9 @@ func TestFilterByReplicationLag(t *testing.T) { } for _, tc := range cases { - lts := make([]*tabletStats, len(tc.input)) + lts := make([]*tabletHealth, len(tc.input)) for i, lag := range tc.input { - lts[i] = &tabletStats{ + lts[i] = &tabletHealth{ Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, @@ -130,52 +130,52 @@ func TestFilterByReplicationLagThreeTabletMin(t *testing.T) { // Use at least 3 tablets if possible testSetMinNumTablets(3) // lags of (1s, 1s, 10m, 11m) - returns at least32 items where the slightly delayed ones that are returned are the 10m and 11m ones. - ts1 := &tabletStats{ + ts1 := &tabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &tabletStats{ + ts2 := &tabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts3 := &tabletStats{ + ts3 := &tabletHealth{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts4 := &tabletStats{ + ts4 := &tabletHealth{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - got := FilterStatsByReplicationLag([]*tabletStats{ts1, ts2, ts3, ts4}) + got := FilterStatsByReplicationLag([]*tabletHealth{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts1) || !got[1].DeepEqual(ts2) || !got[2].DeepEqual(ts3) { t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } // lags of (11m, 10m, 1s, 1s) - reordered tablets returns the same 3 items where the slightly delayed one that is returned is the 10m and 11m ones. - ts1 = &tabletStats{ + ts1 = &tabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - ts2 = &tabletStats{ + ts2 = &tabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts3 = &tabletStats{ + ts3 = &tabletHealth{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts4 = &tabletStats{ + ts4 = &tabletHealth{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - got = FilterStatsByReplicationLag([]*tabletStats{ts1, ts2, ts3, ts4}) + got = FilterStatsByReplicationLag([]*tabletHealth{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts3) || !got[1].DeepEqual(ts4) || !got[2].DeepEqual(ts2) { t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } @@ -187,32 +187,32 @@ func TestFilterStatsByReplicationLagOneTabletMin(t *testing.T) { // Use at least 1 tablets if possible testSetMinNumTablets(1) // lags of (1s, 100m) - return only healthy tablet if that is all that is available. - ts1 := &tabletStats{ + ts1 := &tabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &tabletStats{ + ts2 := &tabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got := FilterStatsByReplicationLag([]*tabletStats{ts1, ts2}) + got := FilterStatsByReplicationLag([]*tabletHealth{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { t.Errorf("FilterStatsByReplicationLag([1s, 100m]) = %+v, want [1s]", got) } // lags of (1m, 100m) - return only healthy tablet if that is all that is healthy enough. - ts1 = &tabletStats{ + ts1 = &tabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60}, } - ts2 = &tabletStats{ + ts2 = &tabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got = FilterStatsByReplicationLag([]*tabletStats{ts1, ts2}) + got = FilterStatsByReplicationLag([]*tabletHealth{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { t.Errorf("FilterStatsByReplicationLag([1m, 100m]) = %+v, want [1m]", got) } @@ -246,12 +246,12 @@ func TestTrivialStatsUpdate(t *testing.T) { } for _, c := range cases { - o := &tabletStats{ + o := &tabletHealth{ Stats: &querypb.RealtimeStats{ SecondsBehindMaster: c.o, }, } - n := &tabletStats{ + n := &tabletHealth{ Stats: &querypb.RealtimeStats{ SecondsBehindMaster: c.n, }, diff --git a/go/vt/discovery/tablet_stats.go b/go/vt/discovery/tablet_health.go similarity index 56% rename from go/vt/discovery/tablet_stats.go rename to go/vt/discovery/tablet_health.go index 3d623426a9a..6d692a37099 100644 --- a/go/vt/discovery/tablet_stats.go +++ b/go/vt/discovery/tablet_health.go @@ -2,8 +2,11 @@ package discovery import ( "bytes" + "context" "fmt" "strings" + "sync" + "vitess.io/vitess/go/vt/vttablet/queryservice" "github.com/golang/protobuf/proto" "vitess.io/vitess/go/netutil" @@ -11,21 +14,17 @@ import ( "vitess.io/vitess/go/vt/proto/topodata" ) -// tabletStats is returned when getting the set of tablets. -type tabletStats struct { - // Key uniquely identifies that serving tablet. It is computed - // from the Tablet's record Hostname and PortMap. If a tablet - // is restarted on different ports, its Key will be different. - // Key is computed using the TabletToMapKey method below. - // key can be used in GetConnection(). - Key string +// TabletHealth maintains the health status of a tablet. A map of this +// structure is maintained in HealthCheckImpl. +type tabletHealth struct { + mu sync.Mutex + // cancelFunc must be called before discarding TabletHealth. + // This will ensure that the associated checkConn goroutine will terminate. + cancelFunc context.CancelFunc + // conn is the connection associated with the tablet. + conn queryservice.QueryService // Tablet is the tablet object that was sent to HealthCheck.AddTablet. Tablet *topodata.Tablet - // Name is an optional tag (e.g. alternative address) for the - // tablet. It is supposed to represent the tablet as a task, - // not as a process. For instance, it can be a - // cell+keyspace+shard+tabletType+taskIndex value. - Name string // Target is the current target as returned by the streaming // StreamHealth RPC. Target *query.Target @@ -46,42 +45,35 @@ type tabletStats struct { LastError error } -// String is defined because we want to print a []*tabletStats array nicely. -func (e *tabletStats) String() string { - return fmt.Sprint(*e) +// String is defined because we want to print a []*tabletHealth array nicely. +func (th *tabletHealth) String() string { + return fmt.Sprintf("TabletHealth{Tablet: %v,Target: %v,Up: %v,Serving: %v, MasterTermStartTime: %v, Stats: %v, LastError: %v", + th.Tablet, th.Target, th.Up, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) } -// DeepEqual compares two tabletStats. Since we include protos, we +// DeepEqual compares two tabletHealth. Since we include protos, we // need to use proto.Equal on these. -func (e *tabletStats) DeepEqual(f *tabletStats) bool { - return e.Key == f.Key && - proto.Equal(e.Tablet, f.Tablet) && - e.Name == f.Name && - proto.Equal(e.Target, f.Target) && - e.Up == f.Up && - e.Serving == f.Serving && - e.MasterTermStartTime == f.MasterTermStartTime && - proto.Equal(e.Stats, f.Stats) && - ((e.LastError == nil && f.LastError == nil) || - (e.LastError != nil && f.LastError != nil && e.LastError.Error() == f.LastError.Error())) -} - -// Copy produces a copy of tabletStats. -func (e *tabletStats) Copy() *tabletStats { - ts := *e - return &ts +func (th *tabletHealth) DeepEqual(other *tabletHealth) bool { + return proto.Equal(th.Tablet, other.Tablet) && + proto.Equal(th.Target, other.Target) && + th.Up == other.Up && + th.Serving == other.Serving && + th.MasterTermStartTime == other.MasterTermStartTime && + proto.Equal(th.Stats, other.Stats) && + ((th.LastError == nil && other.LastError == nil) || + (th.LastError != nil && other.LastError != nil && th.LastError.Error() == other.LastError.Error())) } // GetTabletHostPort formats a tablet host port address. -func (e tabletStats) GetTabletHostPort() string { - vtPort := e.Tablet.PortMap["vt"] - return netutil.JoinHostPort(e.Tablet.Hostname, vtPort) +func (th *tabletHealth) GetTabletHostPort() string { + vtPort := th.Tablet.PortMap["vt"] + return netutil.JoinHostPort(th.Tablet.Hostname, vtPort) } // GetHostNameLevel returns the specified hostname level. If the level does not exist it will pick the closest level. // This seems unused but can be utilized by certain url formatting templates. See getTabletDebugURL for more details. -func (e tabletStats) GetHostNameLevel(level int) string { - chunkedHostname := strings.Split(e.Tablet.Hostname, ".") +func (th *tabletHealth) GetHostNameLevel(level int) string { + chunkedHostname := strings.Split(th.Tablet.Hostname, ".") if level < 0 { return chunkedHostname[0] @@ -95,8 +87,8 @@ func (e tabletStats) GetHostNameLevel(level int) string { // getTabletDebugURL formats a debug url to the tablet. // It uses a format string that can be passed into the app to format // the debug URL to accommodate different network setups. It applies -// the html/template string defined to a tabletStats object. The -// format string can refer to members and functions of tabletStats +// the html/template string defined to a tabletHealth object. The +// format string can refer to members and functions of tabletHealth // like a regular html/template string. // // For instance given a tablet with hostname:port of host.dc.domain:22 @@ -104,19 +96,19 @@ func (e tabletStats) GetHostNameLevel(level int) string { // http://{{.GetTabletHostPort}} -> http://host.dc.domain:22 // https://{{.Tablet.Hostname}} -> https://host.dc.domain // https://{{.GetHostNameLevel 0}}.bastion.corp -> https://host.bastion.corp -func (e tabletStats) getTabletDebugURL() string { +func (th *tabletHealth) getTabletDebugURL() string { var buffer bytes.Buffer - tabletURLTemplate.Execute(&buffer, e) + tabletURLTemplate.Execute(&buffer, th) return buffer.String() } -// TrivialStatsUpdate returns true iff the old and new tabletStats +// TrivialStatsUpdate returns true iff the old and new tabletHealth // haven't changed enough to warrant re-calling FilterLegacyStatsByReplicationLag. -func (e *tabletStats) TrivialStatsUpdate(n *tabletStats) bool { +func (th *tabletHealth) TrivialStatsUpdate(n *tabletHealth) bool { // Skip replag filter when replag remains in the low rep lag range, // which should be the case majority of the time. lowRepLag := lowReplicationLag.Seconds() - oldRepLag := float64(e.Stats.SecondsBehindMaster) + oldRepLag := float64(th.Stats.SecondsBehindMaster) newRepLag := float64(n.Stats.SecondsBehindMaster) if oldRepLag <= lowRepLag && newRepLag <= lowRepLag { return true @@ -134,3 +126,15 @@ func (e *tabletStats) TrivialStatsUpdate(n *tabletStats) bool { return false } + +func (th *tabletHealth) deleteConnLocked() { + th.mu.Lock() + defer th.mu.Unlock() + th.Up = false + th.conn = nil + th.cancelFunc() +} + +func (th *tabletHealth) isHealthy() bool { + return th.Serving && th.LastError == nil && th.Stats != nil && !IsReplicationLagVeryHigh(th) +} diff --git a/go/vt/discovery/tablet_stats_cache.go b/go/vt/discovery/tablet_stats_cache.go deleted file mode 100644 index 0c4826c1cd6..00000000000 --- a/go/vt/discovery/tablet_stats_cache.go +++ /dev/null @@ -1,179 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package discovery - -import ( - "sync" - - querypb "vitess.io/vitess/go/vt/proto/query" - - "vitess.io/vitess/go/vt/log" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/topo/topoproto" -) - -// tabletStatsCache is an internal data structure that keeps both the -// current list of available tabletStats, and a serving list: -// - for master tablets, only the current master is kept. -// - for non-master tablets, we filter the list using FilterStatsByReplicationLag. -// It keeps entries for all tablets in the cell(s) it's configured to serve for, -// and for the master independently of which cell it's in. -// Note the healthy tablet computation is done when we receive a tablet -// update only, not at serving time. -// Also note the cache may not have the last entry received from the tablet. -// For instance, if a tablet was healthy, and is still healthy, we do not -// keep its new update. -type tabletStatsCache struct { - // mu protects the following fields. It does not protect individual - // entries in the entries map. - mu sync.Mutex - // entries maps from keyspace/shard/tabletType to our cache. - entries map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry -} - -func newTabletStatsCache() *tabletStatsCache { - tc := &tabletStatsCache{ - entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), - } - return tc -} - -// getEntry returns an existing TabletStatsCacheEntry in the cache, or nil -// if the entry does not exist. It only takes a Read lock on mu. -func (tc *tabletStatsCache) getEntry(keyspace, shard string, tabletType topodatapb.TabletType) *tabletStatsCacheEntry { - tc.mu.Lock() - defer tc.mu.Unlock() - - if s, ok := tc.entries[keyspace]; ok { - if t, ok := s[shard]; ok { - if e, ok := t[tabletType]; ok { - return e - } - } - } - return nil -} - -// getOrCreateEntry returns an existing TabletStatsCacheEntry from the cache, -// or creates it if it doesn't exist. -func (tc *tabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStatsCacheEntry { - // Fast path (most common path too): Read-lock, return the entry. - if e := tc.getEntry(target.Keyspace, target.Shard, target.TabletType); e != nil { - return e - } - - // Slow path: Lock, will probably have to add the entry at some level. - tc.mu.Lock() - defer tc.mu.Unlock() - - s, ok := tc.entries[target.Keyspace] - if !ok { - s = make(map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry) - tc.entries[target.Keyspace] = s - } - t, ok := s[target.Shard] - if !ok { - t = make(map[topodatapb.TabletType]*tabletStatsCacheEntry) - s[target.Shard] = t - } - e, ok := t[target.TabletType] - if !ok { - e = &tabletStatsCacheEntry{ - all: make(map[string]*tabletStats), - } - t[target.TabletType] = e - } - return e -} - -// tabletStatsCacheEntry is the per keyspace/shard/tabletType -// entry of the in-memory map for tabletStatsCache. -type tabletStatsCacheEntry struct { - // mu protects the rest of this structure. - mu sync.Mutex - // all has the valid tablets, indexed by TabletToMapKey(ts.Tablet), - // as it is the index used by HealthCheck. - all map[string]*tabletStats - // healthy only has the healthy ones. - healthy []*tabletStats -} - -func (e *tabletStatsCacheEntry) updateHealthyMapForMaster(ts *tabletStats) { - if ts.Target.TabletType != topodatapb.TabletType_MASTER { - panic("program bug") - } - if ts.Up { - // We have an Up master. - if len(e.healthy) == 0 { - // We have a new Up server, just remember it. - e.healthy = append(e.healthy, ts) - return - } - - // We already have one up server, see if we - // need to replace it. - if ts.MasterTermStartTime < e.healthy[0].MasterTermStartTime { - log.Warningf("not marking healthy master %s as Up for %s because its externally reparented timestamp is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", - topoproto.TabletAliasString(ts.Tablet.Alias), - topoproto.KeyspaceShardString(ts.Target.Keyspace, ts.Target.Shard), - topoproto.TabletAliasString(e.healthy[0].Tablet.Alias), - ts.MasterTermStartTime, - e.healthy[0].MasterTermStartTime) - return - } - - // Just replace it. - e.healthy[0] = ts - return - } - - // We have a Down master, remove it only if it's exactly the same. - if len(e.healthy) != 0 { - if ts.Key == e.healthy[0].Key { - // Same guy, remove it. - e.healthy = nil - } - } -} - -func (e *tabletStatsCacheEntry) getHealthyTabletStats() []tabletStats { - e.mu.Lock() - defer e.mu.Unlock() - result := make([]tabletStats, len(e.healthy)) - for i, ts := range e.healthy { - result[i] = *ts - } - return result -} - -func (e *tabletStatsCacheEntry) getTabletStats() []tabletStats { - e.mu.Lock() - defer e.mu.Unlock() - result := make([]tabletStats, 0, len(e.all)) - for _, ts := range e.all { - result = append(result, *ts) - } - return result -} - -// ResetForTesting is for use in tests only. -func (tc *tabletStatsCache) ResetForTesting() { - tc.mu.Lock() - defer tc.mu.Unlock() - - tc.entries = make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry) -} diff --git a/go/vt/discovery/tablet_stats_cache_test.go b/go/vt/discovery/tablet_stats_cache_test.go deleted file mode 100644 index 84002dab5b9..00000000000 --- a/go/vt/discovery/tablet_stats_cache_test.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package discovery - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/topo" -) - -// TestTabletStatsCache tests the functionality of the TabletStatsCache class. -func TestTabletStatsCache(t *testing.T) { - // We want to unit test TabletStatsCache without a full-blown - // HealthCheck object, so we can't call NewTabletStatsCache. - // So we just construct this object here. - tsc := &tabletStatsCache{ - entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), - } - - // empty - a := tsc.getEntry("k", "s", topodatapb.TabletType_MASTER) - assert.Nil(t, a) - // add a tablet - b := tsc.getOrCreateEntry(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) - assert.NotNil(t, b) - tablet1 := topo.NewTablet(10, "cell", "host1") - ts1 := &tabletStats{ - Key: "t1", - Tablet: tablet1, - Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, - Up: true, - Serving: true, - Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, - } - b.all[TabletToMapKey(tablet1)] = ts1 - - // check it's there - c := tsc.getOrCreateEntry(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) - assert.NotNil(t, c) - got := c.getTabletStats() - assert.Equal(t, 1, len(got)) - assert.True(t, ts1.DeepEqual(&got[0])) - - // add a second tablet - tablet2 := topo.NewTablet(11, "cell", "host2") - ts2 := &tabletStats{ - Key: "t2", - Tablet: tablet2, - Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, - Up: true, - Serving: true, - Stats: &querypb.RealtimeStats{SecondsBehindMaster: 0, CpuUsage: 0.2}, - } - d := tsc.getOrCreateEntry(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) - d.all[TabletToMapKey(tablet2)] = ts2 - - d.updateHealthyMapForMaster(ts2) - // should be in healthy tablet stats - got = d.getHealthyTabletStats() - assert.Equal(t, 1, len(got)) - assert.True(t, ts2.DeepEqual(&got[0])) - // should be in all tabletStats - got = d.getTabletStats() - assert.Equal(t, 1, len(got)) - assert.True(t, ts2.DeepEqual(&got[0])) - - // master goes down - ts2.Up = false - d.updateHealthyMapForMaster(ts2) - got = d.getHealthyTabletStats() - // check it is not there - assert.Equal(t, 0, len(got)) - -} diff --git a/go/vt/discovery/tablets_cache_status.go b/go/vt/discovery/tablets_cache_status.go new file mode 100644 index 00000000000..4f7cfcd8987 --- /dev/null +++ b/go/vt/discovery/tablets_cache_status.go @@ -0,0 +1,87 @@ +package discovery + +import ( + "fmt" + "html/template" + "sort" + "strings" + + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo/topoproto" +) + +// TabletsCacheStatus is the current tablets for a cell/target. +type TabletsCacheStatus struct { + Cell string + Target *querypb.Target + TabletsStats TabletStatsList +} + +// TabletStatsList is used for sorting. +type TabletStatsList []*tabletHealth + +// Len is part of sort.Interface. +func (tsl TabletStatsList) Len() int { + return len(tsl) +} + +// Less is part of sort.Interface +func (tsl TabletStatsList) Less(i, j int) bool { + name1 := topoproto.TabletAliasString(tsl[i].Tablet.Alias) + name2 := topoproto.TabletAliasString(tsl[j].Tablet.Alias) + return name1 < name2 +} + +// Swap is part of sort.Interface +func (tsl TabletStatsList) Swap(i, j int) { + tsl[i], tsl[j] = tsl[j], tsl[i] +} + +// StatusAsHTML returns an HTML version of the status. +func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML { + tLinks := make([]string, 0, 1) + if tcs.TabletsStats != nil { + sort.Sort(tcs.TabletsStats) + } + for _, ts := range tcs.TabletsStats { + color := "green" + extra := "" + if ts.LastError != nil { + color = "red" + extra = fmt.Sprintf(" (%v)", ts.LastError) + } else if !ts.Serving { + color = "red" + extra = " (Not Serving)" + } else if !ts.Up { + color = "red" + extra = " (Down)" + } else if ts.Target.TabletType == topodatapb.TabletType_MASTER { + extra = fmt.Sprintf(" (MasterTS: %v)", ts.MasterTermStartTime) + } else { + extra = fmt.Sprintf(" (RepLag: %v)", ts.Stats.SecondsBehindMaster) + } + name := ts.GetTabletHostPort() + tLinks = append(tLinks, fmt.Sprintf(`%v%v`, ts.getTabletDebugURL(), color, name, extra)) + } + return template.HTML(strings.Join(tLinks, "
")) +} + +// TabletsCacheStatusList is used for sorting. +type TabletsCacheStatusList []*TabletsCacheStatus + +// Len is part of sort.Interface. +func (tcsl TabletsCacheStatusList) Len() int { + return len(tcsl) +} + +// Less is part of sort.Interface +func (tcsl TabletsCacheStatusList) Less(i, j int) bool { + return tcsl[i].Cell+"."+tcsl[i].Target.Keyspace+"."+tcsl[i].Target.Shard+"."+string(tcsl[i].Target.TabletType) < + tcsl[j].Cell+"."+tcsl[j].Target.Keyspace+"."+tcsl[j].Target.Shard+"."+string(tcsl[j].Target.TabletType) +} + +// Swap is part of sort.Interface +func (tcsl TabletsCacheStatusList) Swap(i, j int) { + tcsl[i], tcsl[j] = tcsl[j], tcsl[i] +} diff --git a/go/vt/discovery/utils.go b/go/vt/discovery/utils.go index 54858b3a574..7a9e94f019b 100644 --- a/go/vt/discovery/utils.go +++ b/go/vt/discovery/utils.go @@ -16,14 +16,6 @@ limitations under the License. package discovery -import ( - "sort" - "strings" - - "vitess.io/vitess/go/netutil" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - // This file contains helper filter methods to process the unfiltered list of // tablets returned by LegacyHealthCheck.GetTabletStatsFrom*. // See also legacy_replicationlag.go for a more sophisicated filter used by vtgate. @@ -45,15 +37,3 @@ func RemoveUnhealthyTablets(tabletStatsList []LegacyTabletStats) []LegacyTabletS } return result } - -// TabletToMapKey creates a key to the map from tablet's host and ports. -// It should only be used in discovery and related module. -func TabletToMapKey(tablet *topodatapb.Tablet) string { - parts := make([]string, 0, 1) - for name, port := range tablet.PortMap { - parts = append(parts, netutil.JoinHostPort(name, port)) - } - sort.Strings(parts) - parts = append([]string{tablet.Hostname}, parts...) - return strings.Join(parts, ",") -} diff --git a/go/vt/vtgate/api.go b/go/vt/vtgate/api.go index 269da43d903..46b25ce301b 100644 --- a/go/vt/vtgate/api.go +++ b/go/vt/vtgate/api.go @@ -103,7 +103,7 @@ func initAPI(ctx context.Context, hc discovery.HealthCheck) { return cacheStatus, nil } if len(parts) != 2 { - return nil, fmt.Errorf("invalid health-check path: %q expected path: / or /cell/ or /keyspace/ or /tablet/", itemPath) + return nil, fmt.Errorf("invalid health-check path: %q expected path: / or /cell/ or /keyspace/ or /tablet/mysql_hostname", itemPath) } value := parts[1] @@ -133,7 +133,7 @@ func initAPI(ctx context.Context, hc discovery.HealthCheck) { // Return a _specific tablet_ for _, tabletCacheStatus := range cacheStatus { for _, tabletStats := range tabletCacheStatus.TabletsStats { - if tabletStats.Name == value || tabletStats.Tablet.MysqlHostname == value { + if tabletStats.Tablet.MysqlHostname == value { return tabletStats, nil } } diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index f2eee9a5bf3..42061f25909 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -89,8 +89,6 @@ func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string statusAggregators: make(map[string]*TabletStatusAggregator), buffer: buffer.New(), } - // Start the healthcheck - hc.Open() gw.QueryService = queryservice.Wrap(nil, gw.withRetry) return gw } From 8e2b51853aaa80324ce17add45607212b14623e9 Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 27 Apr 2020 21:16:49 -0700 Subject: [PATCH 08/39] healthcheck: if cells_to_watch is empty, operate on just localCell Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index be37dea588f..c2e1e015a95 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -248,11 +248,14 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur var topoWatchers []*TopologyWatcher var filter TabletFilter - for _, c := range strings.Split(*CellsToWatch, ",") { + cells := strings.Split(*CellsToWatch, ",") + if len(cells) == 0 { + cells = append(cells, localCell) + } + for _, c := range cells { if c == "" { continue } - // very simplistic, assumes localCell is not given as part of *CellsToWatch if len(TabletFilters) > 0 { if len(KeyspacesToWatch) > 0 { log.Exitf("Only one of -keyspaces_to_watch and -tablet_filters may be specified at a time") From 5808934bf9bb1101d874f40fe9f00b36b4eb9378 Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 27 Apr 2020 21:35:55 -0700 Subject: [PATCH 09/39] healthcheck: delete commented code Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 107 --------------------------------- 1 file changed, 107 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index c2e1e015a95..78d3857c0ba 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -516,113 +516,6 @@ func (hc *HealthCheckImpl) stateChecksum() int64 { return int64(crc32.ChecksumIEEE(buf.Bytes())) } -/* -// updateHealth updates the TabletHealth record and updates the tablet stats -func (hc *HealthCheckImpl) updateHealth(ts *tabletStats, conn queryservice.QueryService) { - // update the stats cache - hc.updateStatsCache(ts) - - hc.mu.Lock() - th, ok := hc.addrToHealth[ts.Key] - if !ok { - // This can happen on delete because the entry is removed first, - // or if HealthCheckImpl has been closed. - hc.mu.Unlock() - return - } - - oldts := th.latestTabletStats - th.latestTabletStats = *ts.Copy() - th.conn = conn - hc.mu.Unlock() - - // In the case where a tablet changes type (but not for the - // initial message), we want to log it, and maybe advertise it too. - if oldts.Target.TabletType != topodatapb.TabletType_UNKNOWN && oldts.Target.TabletType != ts.Target.TabletType { - // Log and maybe notify - log.Infof("HealthCheckUpdate(Type Change): %v, tablet: %s, target %+v => %+v, reparent time: %v", - oldts.Name, topotools.TabletIdent(oldts.Tablet), topotools.TargetIdent(oldts.Target), topotools.TargetIdent(ts.Target), ts.MasterTermStartTime) - oldts.Up = false - hc.updateStatsCache(&oldts) - - // Track how often a tablet gets promoted to master. It is used for - // comparing against the variables in go/vtgate/buffer/variables.go. - if oldts.Target.TabletType != topodatapb.TabletType_MASTER && ts.Target.TabletType == topodatapb.TabletType_MASTER { - hcMasterPromotedCounters.Add([]string{ts.Target.Keyspace, ts.Target.Shard}, 1) - } - } -} - -func (hc *HealthCheckImpl) updateStatsCache(stats *tabletStats) { - if stats.Target.TabletType != topodatapb.TabletType_MASTER && - stats.Tablet.Alias.Cell != hc.cell && - hc.getAliasByCell(stats.Tablet.Alias.Cell) != hc.getAliasByCell(hc.cell) { - // this is for a non-master tablet in a different cell and a different alias, drop it - return - } - - // We assume that if we are getting an update for a tablet, then we are interested in that tablet - // i.e. it belongs to a cell we are watching. - tsc := hc.tscByCell[stats.Tablet.Alias.Cell] - e := tsc.getOrCreateEntry(stats.Target) - e.mu.Lock() - defer e.mu.Unlock() - - // Update our full map. - trivialNonMasterUpdate := false - current, exists := e.all[stats.Key] - - switch { - case exists && stats.Up: - // We have an current entry, and a new entry. - // Remember if they are both good (most common case). - trivialNonMasterUpdate = current.LastError == nil && current.Serving && stats.LastError == nil && - stats.Serving && stats.Target.TabletType != topodatapb.TabletType_MASTER && current.TrivialStatsUpdate(stats) - - // We already have the entry, update the - // values if necessary. (will update both - // 'all' and 'healthy' as they use pointers). - if !trivialNonMasterUpdate { - *current = *stats - } - case exists && !stats.Up: - // We have an entry which we shouldn't. Remove it. - delete(e.all, stats.Key) - case !exists && stats.Up: - // Add the entry. - e.all[stats.Key] = stats - case !exists && !stats.Up: - // We were told to remove an entry which we - // didn't have anyway, nothing should happen. - return - default: - // Unreachable - } - - // Update our healthy list. - var allArray []*tabletStats - if stats.Target.TabletType == topodatapb.TabletType_MASTER { - // The healthy list is different for TabletType_MASTER: we - // only keep the most recent one. - e.updateHealthyMapForMaster(stats) - } else { - // For non-master, if it is a trivial update, - // we just skip everything else. We don't even update the - // aggregate stats. - if trivialNonMasterUpdate { - return - } - - // Now we need to do some work. Recompute our healthy list. - allArray = make([]*tabletStats, 0, len(e.all)) - for _, s := range e.all { - allArray = append(allArray, s) - } - e.healthy = FilterStatsByReplicationLag(allArray) - } -} -*/ - // finalizeConn closes the health checking connection and sends the final // notification about the tablet to downstream. To be called only on exit from // checkConn(). From 3d953190b5cf1eed805e12c2399e3baa8a1d786a Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 28 Apr 2020 13:07:47 -0700 Subject: [PATCH 10/39] healthcheck: use tabletHealth.mu to serialize access to members. handle tabletType change in healthcheck Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 34 ++++++++++++++++---- go/vt/discovery/replicationlag_test.go | 43 -------------------------- go/vt/discovery/tablet_health.go | 38 +++++++---------------- 3 files changed, 39 insertions(+), 76 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 78d3857c0ba..78de594d14f 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -520,6 +520,8 @@ func (hc *HealthCheckImpl) stateChecksum() int64 { // notification about the tablet to downstream. To be called only on exit from // checkConn(). func (hc *HealthCheckImpl) finalizeConn(hcc *healthCheckConn) { + hcc.tabletHealth.mu.Lock() + defer hcc.tabletHealth.mu.Unlock() hcc.tabletHealth.Up = false hcc.setServingState(false, "finalizeConn closing connection") // Note: checkConn() exits only when hcc.ctx.Done() is closed. Thus it's @@ -589,9 +591,11 @@ func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn) { // This will ensure that this update prevails over any previous message that // stream could have sent. if timedout.Get() { + hcc.tabletHealth.mu.Lock() hcc.tabletHealth.LastError = fmt.Errorf("healthcheck timed out (latest %v)", hcc.lastResponseTimestamp) hcc.setServingState(false, hcc.tabletHealth.LastError.Error()) hcErrorCounters.Add([]string{hcc.tabletHealth.Target.Keyspace, hcc.tabletHealth.Target.Shard, topoproto.TabletTypeLString(hcc.tabletHealth.Target.TabletType)}, 1) + hcc.tabletHealth.mu.Unlock() } // Streaming RPC failed e.g. because vttablet was restarted or took too long. @@ -616,7 +620,7 @@ func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn) { // from the health check connection are logged the first time, // but don't continue to log if the connection stays down. // -// hcc.mu must be locked before calling this function +// hcc.tabletHealth.mu must be locked before calling this function func (hcc *healthCheckConn) setServingState(serving bool, reason string) { if !hcc.loggedServingState || (serving != hcc.tabletHealth.Serving) { // Emit the log from a separate goroutine to avoid holding @@ -636,22 +640,28 @@ func (hcc *healthCheckConn) setServingState(serving bool, reason string) { // stream streams healthcheck responses to callback. func (hcc *healthCheckConn) stream(ctx context.Context, callback func(*querypb.StreamHealthResponse) error) { + hcc.tabletHealth.mu.Lock() if hcc.tabletHealth.conn == nil { conn, err := tabletconn.GetDialer()(hcc.tabletHealth.Tablet, grpcclient.FailFast(true)) if err != nil { hcc.tabletHealth.LastError = err + hcc.tabletHealth.mu.Unlock() return } hcc.tabletHealth.conn = conn hcc.tabletHealth.LastError = nil } + conn := hcc.tabletHealth.conn + hcc.tabletHealth.mu.Unlock() - if err := hcc.tabletHealth.conn.StreamHealth(ctx, callback); err != nil { + if err := conn.StreamHealth(ctx, callback); err != nil { + hcc.tabletHealth.mu.Lock() log.Warningf("tablet %v healthcheck stream error: %v", hcc.tabletHealth.Tablet.Alias, err) hcc.setServingState(false, err.Error()) hcc.tabletHealth.LastError = err hcc.tabletHealth.conn.Close(ctx) hcc.tabletHealth.conn = nil + hcc.tabletHealth.mu.Unlock() } } @@ -676,21 +686,32 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St serving = false } - // hcc.tabletHealth.Tablet.Alias.Uid may be 0 because the youtube internal mechanism uses a different - // code path to initialize this value. If so, we should skip this check. - if shr.TabletAlias != nil && hcc.tabletHealth.Tablet.Alias.Uid != 0 && !proto.Equal(shr.TabletAlias, hcc.tabletHealth.Tablet.Alias) { + if shr.TabletAlias != nil && !proto.Equal(shr.TabletAlias, hcc.tabletHealth.Tablet.Alias) { return fmt.Errorf("health stats mismatch, tablet %+v alias does not match response alias %v", hcc.tabletHealth.Tablet, shr.TabletAlias) } + hcc.tabletHealth.mu.Lock() + currentTablet := hcc.tabletHealth.Tablet + hcc.tabletHealth.mu.Unlock() // In this case where a new tablet is initialized or a tablet type changes, we want to // initialize the counter so the rate can be calculated correctly. - if hcc.tabletHealth.Target.TabletType != shr.Target.TabletType { + if currentTablet.Type != shr.Target.TabletType { hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) + // hc still has this tabletHealth in the wrong target (because tabletType changed) + oldTargetKey := hc.keyFromTablet(currentTablet) + newTargetKey := hc.keyFromTarget(shr.Target) + tabletAlias := topoproto.TabletAliasString(currentTablet.Alias) + hc.mu.Lock() + delete(hc.entries[oldTargetKey], tabletAlias) + hc.entries[newTargetKey][tabletAlias] = hcc.tabletHealth + hc.mu.Unlock() } // Update our record, and notify downstream for tabletType and // realtimeStats change. hcc.lastResponseTimestamp = time.Now() + hcc.tabletHealth.mu.Lock() + defer hcc.tabletHealth.mu.Unlock() hcc.tabletHealth.Target = shr.Target hcc.tabletHealth.MasterTermStartTime = shr.TabletExternallyReparentedTimestamp hcc.tabletHealth.Stats = shr.RealtimeStats @@ -717,6 +738,7 @@ func (hc *HealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { th, ok := ths[tabletAlias] if !ok { log.Warningf("Something is wrong, we have no health data for tablet: %v", tabletAlias) + return } th.deleteConnLocked() delete(ths, tabletAlias) diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index 79aedc30d7c..77a01c822ed 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -219,46 +219,3 @@ func TestFilterStatsByReplicationLagOneTabletMin(t *testing.T) { // Reset to the default testSetMinNumTablets(2) } - -func TestTrivialStatsUpdate(t *testing.T) { - // Note the healthy threshold is set to 30s. - cases := []struct { - o uint32 - n uint32 - expected bool - }{ - // both are under 30s - {o: 0, n: 1, expected: true}, - {o: 15, n: 20, expected: true}, - - // one is under 30s, the other isn't - {o: 2, n: 40, expected: false}, - {o: 40, n: 10, expected: false}, - - // both are over 30s, but close enough - {o: 100, n: 100, expected: true}, - {o: 100, n: 105, expected: true}, - {o: 105, n: 100, expected: true}, - - // both are over 30s, but too far - {o: 100, n: 120, expected: false}, - {o: 120, n: 100, expected: false}, - } - - for _, c := range cases { - o := &tabletHealth{ - Stats: &querypb.RealtimeStats{ - SecondsBehindMaster: c.o, - }, - } - n := &tabletHealth{ - Stats: &querypb.RealtimeStats{ - SecondsBehindMaster: c.n, - }, - } - got := o.TrivialStatsUpdate(n) - if got != c.expected { - t.Errorf("TrivialStatsUpdate(%v, %v) = %v, expected %v", c.o, c.n, got, c.expected) - } - } -} diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 6d692a37099..df1031036cf 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "sync" + "vitess.io/vitess/go/vt/vttablet/queryservice" "github.com/golang/protobuf/proto" @@ -47,6 +48,8 @@ type tabletHealth struct { // String is defined because we want to print a []*tabletHealth array nicely. func (th *tabletHealth) String() string { + th.mu.Lock() + defer th.mu.Unlock() return fmt.Sprintf("TabletHealth{Tablet: %v,Target: %v,Up: %v,Serving: %v, MasterTermStartTime: %v, Stats: %v, LastError: %v", th.Tablet, th.Target, th.Up, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) } @@ -66,14 +69,20 @@ func (th *tabletHealth) DeepEqual(other *tabletHealth) bool { // GetTabletHostPort formats a tablet host port address. func (th *tabletHealth) GetTabletHostPort() string { + th.mu.Lock() + hostname := th.Tablet.Hostname vtPort := th.Tablet.PortMap["vt"] - return netutil.JoinHostPort(th.Tablet.Hostname, vtPort) + th.mu.Unlock() + return netutil.JoinHostPort(hostname, vtPort) } // GetHostNameLevel returns the specified hostname level. If the level does not exist it will pick the closest level. // This seems unused but can be utilized by certain url formatting templates. See getTabletDebugURL for more details. func (th *tabletHealth) GetHostNameLevel(level int) string { - chunkedHostname := strings.Split(th.Tablet.Hostname, ".") + th.mu.Lock() + hostname := th.Tablet.Hostname + th.mu.Unlock() + chunkedHostname := strings.Split(hostname, ".") if level < 0 { return chunkedHostname[0] @@ -102,31 +111,6 @@ func (th *tabletHealth) getTabletDebugURL() string { return buffer.String() } -// TrivialStatsUpdate returns true iff the old and new tabletHealth -// haven't changed enough to warrant re-calling FilterLegacyStatsByReplicationLag. -func (th *tabletHealth) TrivialStatsUpdate(n *tabletHealth) bool { - // Skip replag filter when replag remains in the low rep lag range, - // which should be the case majority of the time. - lowRepLag := lowReplicationLag.Seconds() - oldRepLag := float64(th.Stats.SecondsBehindMaster) - newRepLag := float64(n.Stats.SecondsBehindMaster) - if oldRepLag <= lowRepLag && newRepLag <= lowRepLag { - return true - } - - // Skip replag filter when replag remains in the high rep lag range, - // and did not change beyond +/- 10%. - // when there is a high rep lag, it takes a long time for it to reduce, - // so it is not necessary to re-calculate every time. - // In that case, we won't save the new record, so we still - // remember the original replication lag. - if oldRepLag > lowRepLag && newRepLag > lowRepLag && newRepLag < oldRepLag*1.1 && newRepLag > oldRepLag*0.9 { - return true - } - - return false -} - func (th *tabletHealth) deleteConnLocked() { th.mu.Lock() defer th.mu.Unlock() From 84f21ab40af495b1e31c9740dcb80bbf4549cbb7 Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 28 Apr 2020 16:24:29 -0700 Subject: [PATCH 11/39] healthcheck: move shuffleTablets back to gateway Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 176 +++++++----------------- go/vt/discovery/healthcheck_test.go | 4 +- go/vt/discovery/replicationlag.go | 16 +-- go/vt/discovery/replicationlag_test.go | 42 +++--- go/vt/discovery/tablet_health.go | 30 ++-- go/vt/discovery/tablets_cache_status.go | 2 +- go/vt/vtgate/tabletgateway.go | 88 ++++++++++-- 7 files changed, 176 insertions(+), 182 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 78de594d14f..3db81f4e49f 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -19,16 +19,13 @@ limitations under the License. // // Use the HealthCheck object to query for tablets and their health. // -// For an example how to use the HealthCheck object, see worker/topo_utils.go. +// For an example how to use the HealthCheck object, see vtgate/tabletgateway.go // // Tablets have to be manually added to the HealthCheck using AddTablet(). // Alternatively, use a Watcher implementation which will constantly watch // a source (e.g. the topology) and add and remove tablets as they are // added or removed from the source. -// For a Watcher example have a look at NewShardReplicationWatcher(). -// -// tabletStatsCache is one implementation, that caches the known tablets -// and the healthy ones per keyspace/shard/tabletType. +// For a Watcher example have a look at NewCellTabletsWatcher(). // // Internally, the HealthCheck module is connected to each tablet and has a // streaming RPC (StreamHealth) open to receive periodic health infos. @@ -42,7 +39,6 @@ import ( "fmt" "hash/crc32" "html/template" - "math/rand" "net/http" "sort" "strings" @@ -51,9 +47,6 @@ import ( "vitess.io/vitess/go/flagutil" - vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" - "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/topo" "github.com/golang/protobuf/proto" @@ -81,15 +74,14 @@ var ( //TODO(deepthi): change these vars back to unexported when discoveryGateway is removed - // CellsToWatch is the list of cells this healthcheck operates over + // CellsToWatch is the list of cells the healthcheck operates over. If it is empty, only the local cell is watched CellsToWatch = flag.String("cells_to_watch", "", "comma-separated list of cells for watching tablets") // AllowedTabletTypes is the list of allowed tablet types. e.g. {MASTER, REPLICA} AllowedTabletTypes []topodatapb.TabletType // TabletFilters are the keyspace|shard or keyrange filters to apply to the full set of tablets TabletFilters flagutil.StringListValue // KeyspacesToWatch - if provided this specifies which keyspaces should be - // visible to a vtgate. By default the vtgate will allow access to any - // keyspace. + // visible to the healthcheck. By default the healthcheck will watch all keyspaces. KeyspacesToWatch flagutil.StringListValue // RefreshInterval is the interval at which healthcheck refreshes its list of tablets from topo RefreshInterval = flag.Duration("tablet_refresh_interval", 1*time.Minute, "tablet refresh interval") @@ -104,9 +96,9 @@ const ( DefaultHealthCheckRetryDelay = 5 * time.Second DefaultHealthCheckTimeout = 1 * time.Minute - // DefaultTopoReadConcurrency can be used as default value for the TopoReadConcurrency parameter of a TopologyWatcher. + // DefaultTopoReadConcurrency is used as the default value for the TopoReadConcurrency parameter of a TopologyWatcher. DefaultTopoReadConcurrency int = 5 - // DefaultTopologyWatcherRefreshInterval can be used as the default value for + // DefaultTopologyWatcherRefreshInterval is used as the default value for // the refresh interval of a topology watcher. DefaultTopologyWatcherRefreshInterval = 1 * time.Minute // HealthCheckTemplate is the HTML code to display a TabletsCacheStatusList @@ -129,7 +121,7 @@ const ( Keyspace Shard TabletType - tabletHealth + TabletHealth {{range $i, $ts := .}} @@ -183,16 +175,17 @@ type HealthCheck interface { CacheStatus() TabletsCacheStatusList // Close stops the healthcheck. Close() error - // GetTabletAndConnection gets a tablet and connection to execute a query on - GetTabletAndConnection(target *querypb.Target, localCell string, invalidTablets map[string]bool) (string, queryservice.QueryService, error) - // WaitForAllServingTablets + // GetHealthyTabletStatts + GetHealthyTabletStats(target *querypb.Target) []*TabletHealth + // WaitForAllServingTablets allows vtgate to wait for all tablets to be serving before accepting requests WaitForAllServingTablets(ctx context.Context, targets []*querypb.Target) error } -// HealthCheckImpl performs health checking and notifies downstream components about any changes. -// It contains a map of TabletHealth objects, each of which stores the health information for -// a tablet. A checkConn goroutine is spawned for each TabletHealth, which is responsible for -// keeping that TabletHealth up-to-date. This is done through callbacks to updateHealth. +// HealthCheckImpl performs health checking and stores the results. +// It contains a map of TabletHealth objects per Target. +// Each TabletHealth object stores the health information for one tablet. +// A checkConn goroutine is spawned for each TabletHealth, which is responsible for +// keeping that TabletHealth up-to-date. // If checkConn terminates for any reason, it updates TabletHealth.Up as false. If a TabletHealth // gets removed from the map, its cancelFunc gets called, which ensures that the associated // checkConn goroutine eventually terminates. @@ -205,17 +198,15 @@ type HealthCheckImpl struct { // mu protects all the following fields. mu sync.Mutex - // TODO(deepthi): verify all access to following fields is actually being protected by mu - // if not needed, move them up - // a map keyed by keyspace.shard.tabletType - // contains a map of tabletHealth keyed by tablet alias for each tablet relevant to the keyspace.shard.tabletType + // contains a map of TabletHealth keyed by tablet alias for each tablet relevant to the keyspace.shard.tabletType // TODO should we include cell in key? - entries map[string]map[string]*tabletHealth + entries map[string]map[string]*TabletHealth // connsWG keeps track of all launched Go routines that monitor tablet connections. connsWG sync.WaitGroup + // topology watchers that inform healthcheck of tablets being added and deleted topoWatchers []*TopologyWatcher // cellAliases is a cache of cell aliases @@ -225,11 +216,12 @@ type HealthCheckImpl struct { // HealthCheckConn is a structure that lives within the scope of // the checkConn goroutine to maintain its internal state. Therefore, // it does not require synchronization. Changes that are relevant to -// healthcheck are transmitted through calls to HealthCheckImpl.updateHealth. +// healthcheck are transmitted through changes to the TabletHealth +// object, which has its own mutex. type healthCheckConn struct { ctx context.Context - tabletHealth *tabletHealth + tabletHealth *TabletHealth loggedServingState bool lastResponseTimestamp time.Time // timestamp of the last healthcheck response } @@ -243,6 +235,10 @@ type healthCheckConn struct { // The duration for which we consider a health check response to be 'fresh'. If we don't get // a health check response from a tablet for more than this duration, we consider the tablet // not healthy. +// topoServer. +// The topology server that this healthcheck object can use to retrieve cell or tablet information +// localCell. +// The localCell for this healthcheck func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string) HealthCheck { log.Infof("loading tablets for cells: %v", *CellsToWatch) @@ -527,13 +523,13 @@ func (hc *HealthCheckImpl) finalizeConn(hcc *healthCheckConn) { // Note: checkConn() exits only when hcc.ctx.Done() is closed. Thus it's // safe to simply get Err() value here and assign to LastError. hcc.tabletHealth.LastError = hcc.ctx.Err() - if hcc.tabletHealth.conn != nil { + if hcc.tabletHealth.Conn != nil { // Don't use hcc.ctx because it's already closed. // Use a separate context, and add a timeout to prevent unbounded waits. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - hcc.tabletHealth.conn.Close(ctx) - hcc.tabletHealth.conn = nil + hcc.tabletHealth.Conn.Close(ctx) + hcc.tabletHealth.Conn = nil } } @@ -620,7 +616,7 @@ func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn) { // from the health check connection are logged the first time, // but don't continue to log if the connection stays down. // -// hcc.tabletHealth.mu must be locked before calling this function +// hcc.TabletHealth.mu must be locked before calling this function func (hcc *healthCheckConn) setServingState(serving bool, reason string) { if !hcc.loggedServingState || (serving != hcc.tabletHealth.Serving) { // Emit the log from a separate goroutine to avoid holding @@ -641,17 +637,17 @@ func (hcc *healthCheckConn) setServingState(serving bool, reason string) { // stream streams healthcheck responses to callback. func (hcc *healthCheckConn) stream(ctx context.Context, callback func(*querypb.StreamHealthResponse) error) { hcc.tabletHealth.mu.Lock() - if hcc.tabletHealth.conn == nil { + if hcc.tabletHealth.Conn == nil { conn, err := tabletconn.GetDialer()(hcc.tabletHealth.Tablet, grpcclient.FailFast(true)) if err != nil { hcc.tabletHealth.LastError = err hcc.tabletHealth.mu.Unlock() return } - hcc.tabletHealth.conn = conn + hcc.tabletHealth.Conn = conn hcc.tabletHealth.LastError = nil } - conn := hcc.tabletHealth.conn + conn := hcc.tabletHealth.Conn hcc.tabletHealth.mu.Unlock() if err := conn.StreamHealth(ctx, callback); err != nil { @@ -659,8 +655,8 @@ func (hcc *healthCheckConn) stream(ctx context.Context, callback func(*querypb.S log.Warningf("tablet %v healthcheck stream error: %v", hcc.tabletHealth.Tablet.Alias, err) hcc.setServingState(false, err.Error()) hcc.tabletHealth.LastError = err - hcc.tabletHealth.conn.Close(ctx) - hcc.tabletHealth.conn = nil + hcc.tabletHealth.Conn.Close(ctx) + hcc.tabletHealth.Conn = nil hcc.tabletHealth.mu.Unlock() } } @@ -697,7 +693,7 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St // initialize the counter so the rate can be calculated correctly. if currentTablet.Type != shr.Target.TabletType { hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) - // hc still has this tabletHealth in the wrong target (because tabletType changed) + // hc still has this TabletHealth in the wrong target (because tabletType changed) oldTargetKey := hc.keyFromTablet(currentTablet) newTargetKey := hc.keyFromTarget(shr.Target) tabletAlias := topoproto.TabletAliasString(currentTablet.Alias) @@ -762,7 +758,7 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet) { } hcc := &healthCheckConn{ ctx: ctx, - tabletHealth: &tabletHealth{ + tabletHealth: &TabletHealth{ cancelFunc: cancelFunc, Tablet: tablet, Target: target, @@ -774,7 +770,7 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet) { key := hc.keyFromTarget(target) tabletAlias := topoproto.TabletAliasString(tablet.Alias) if ths, ok := hc.entries[key]; !ok { - hc.entries[key] = make(map[string]*tabletHealth) + hc.entries[key] = make(map[string]*TabletHealth) hc.entries[key][tabletAlias] = hcc.tabletHealth } else { if _, ok := ths[tabletAlias]; !ok { @@ -799,22 +795,22 @@ func (hc *HealthCheckImpl) ReplaceTablet(old, new *topodatapb.Tablet) { hc.AddTablet(new) } -// getConnection returns the TabletConn of the given tablet. -func (hc *HealthCheckImpl) getConnection(key string) queryservice.QueryService { +// GetConnection returns the TabletConn of the given tablet. +func (hc *HealthCheckImpl) GetConnection(tabletAlias string) queryservice.QueryService { hc.mu.Lock() defer hc.mu.Unlock() - th := hc.findTabletHealthByAlias(key) + th := hc.findTabletHealthByAlias(tabletAlias) if th == nil { return nil } - return th.conn + return th.Conn } -func (hc *HealthCheckImpl) findTabletHealthByAlias(key string) *tabletHealth { +func (hc *HealthCheckImpl) findTabletHealthByAlias(alias string) *TabletHealth { for _, ths := range hc.entries { for _, th := range ths { - if topoproto.TabletAliasString(th.Tablet.Alias) == key { + if topoproto.TabletAliasString(th.Tablet.Alias) == alias { return th } } @@ -900,39 +896,12 @@ func (hc *HealthCheckImpl) topologyWatcherChecksum() int64 { return checksum } -// GetTabletAndConnection gets you a tablet connection and it's "Key" as produced by TabletToMapKey -// The Key is used by the caller to keep track of invalidTablets -func (hc *HealthCheckImpl) GetTabletAndConnection(target *querypb.Target, localCell string, invalidTablets map[string]bool) (string, queryservice.QueryService, error) { - tablets := hc.getHealthyTabletStats(target) - if len(tablets) == 0 { - // fail fast if there is no tablet - err := vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no valid tablet") - return "", nil, err - } - hc.shuffleTablets(localCell, tablets) - - // skip tablets we tried before - for _, t := range tablets { - tabletAlias := hc.keyFromTablet(t.Tablet) - if _, ok := invalidTablets[tabletAlias]; !ok { - conn := hc.getConnection(tabletAlias) - if conn == nil { - invalidTablets[tabletAlias] = true - } else { - return tabletAlias, conn, nil - } - } - } - err := vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no available connection") - return "", nil, err -} - // GetHealthyTabletStats returns only the healthy targets. // The returned array is owned by the caller. // For TabletType_MASTER, this will only return at most one entry, // the most recent tablet of type master. -func (hc *HealthCheckImpl) getHealthyTabletStats(target *querypb.Target) []*tabletHealth { - var result []*tabletHealth +func (hc *HealthCheckImpl) GetHealthyTabletStats(target *querypb.Target) []*TabletHealth { + var result []*TabletHealth // we check all tablet types in all cells because of cellAliases key := hc.keyFromTarget(target) ths, ok := hc.entries[key] @@ -960,8 +929,8 @@ func (hc *HealthCheckImpl) getHealthyTabletStats(target *querypb.Target) []*tabl // The returned array is owned by the caller. // For TabletType_MASTER, this will only return at most one entry, // the most recent tablet of type master. -func (hc *HealthCheckImpl) getTabletStats(target *querypb.Target) []*tabletHealth { - var result []*tabletHealth +func (hc *HealthCheckImpl) getTabletStats(target *querypb.Target) []*TabletHealth { + var result []*TabletHealth // we check all tablet types in all cells because of cellAliases for _, ths := range hc.entries { for _, th := range ths { @@ -971,53 +940,6 @@ func (hc *HealthCheckImpl) getTabletStats(target *querypb.Target) []*tabletHealt return result } -func (hc *HealthCheckImpl) shuffleTablets(cell string, tablets []*tabletHealth) { - sameCell, diffCell, sameCellMax := 0, 0, -1 - length := len(tablets) - - // move all same cell tablets to the front, this is O(n) - for { - sameCellMax = diffCell - 1 - sameCell = hc.nextTablet(cell, tablets, sameCell, length, true) - diffCell = hc.nextTablet(cell, tablets, diffCell, length, false) - // either no more diffs or no more same cells should stop the iteration - if sameCell < 0 || diffCell < 0 { - break - } - - if sameCell < diffCell { - // fast forward the `sameCell` lookup to `diffCell + 1`, `diffCell` unchanged - sameCell = diffCell + 1 - } else { - // sameCell > diffCell, swap needed - tablets[sameCell], tablets[diffCell] = tablets[diffCell], tablets[sameCell] - sameCell++ - diffCell++ - } - } - - //shuffle in same cell tablets - for i := sameCellMax; i > 0; i-- { - swap := rand.Intn(i + 1) - tablets[i], tablets[swap] = tablets[swap], tablets[i] - } - - //shuffle in diff cell tablets - for i, diffCellMin := length-1, sameCellMax+1; i > diffCellMin; i-- { - swap := rand.Intn(i-sameCellMax) + diffCellMin - tablets[i], tablets[swap] = tablets[swap], tablets[i] - } -} - -func (hc *HealthCheckImpl) nextTablet(cell string, tablets []*tabletHealth, offset, length int, sameCell bool) int { - for ; offset < length; offset++ { - if (tablets[offset].Tablet.Alias.Cell == cell) == sameCell { - return offset - } - } - return -1 -} - func (hc *HealthCheckImpl) getAliasByCell(cell string) string { hc.mu.Lock() defer hc.mu.Unlock() @@ -1077,9 +999,9 @@ func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*queryp continue } - var stats []*tabletHealth + var stats []*TabletHealth if requireServing { - stats = hc.getHealthyTabletStats(target) + stats = hc.GetHealthyTabletStats(target) } else { stats = hc.getTabletStats(target) } diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go index b144e6451cd..d91c640bf1f 100644 --- a/go/vt/discovery/healthcheck_test.go +++ b/go/vt/discovery/healthcheck_test.go @@ -130,7 +130,7 @@ func TestHealthCheckTimeout(t *testing.T) { func TestTemplate(t *testing.T) { tablet := topo.NewTablet(0, "cell", "a") - ts := []*tabletHealth{ + ts := []*TabletHealth{ { Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -161,7 +161,7 @@ func TestDebugURLFormatting(t *testing.T) { ParseTabletURLTemplateFromFlag() tablet := topo.NewTablet(0, "cell", "host.dc.domain") - ts := []*tabletHealth{ + ts := []*TabletHealth{ { Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index c1257d584d5..c47d6740ae8 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -31,18 +31,18 @@ var ( // IsReplicationLagHigh verifies that the given LegacytabletHealth refers to a tablet with high // replication lag, i.e. higher than the configured discovery_low_replication_lag flag. -func IsReplicationLagHigh(tabletHealth *tabletHealth) bool { +func IsReplicationLagHigh(tabletHealth *TabletHealth) bool { return float64(tabletHealth.Stats.SecondsBehindMaster) > lowReplicationLag.Seconds() } // IsReplicationLagVeryHigh verifies that the given LegacytabletHealth refers to a tablet with very high // replication lag, i.e. higher than the configured discovery_high_replication_lag_minimum_serving flag. -func IsReplicationLagVeryHigh(tabletHealth *tabletHealth) bool { +func IsReplicationLagVeryHigh(tabletHealth *TabletHealth) bool { return float64(tabletHealth.Stats.SecondsBehindMaster) > highReplicationLagMinServing.Seconds() } -// FilterStatsByReplicationLag filters the list of tabletHealth by tabletHealth.Stats.SecondsBehindMaster. -// Note that tabletHealth that is non-serving or has error is ignored. +// FilterStatsByReplicationLag filters the list of TabletHealth by TabletHealth.Stats.SecondsBehindMaster. +// Note that TabletHealth that is non-serving or has error is ignored. // // The simplified logic: // - Return tablets that have lag <= lowReplicationLag. @@ -66,11 +66,11 @@ func IsReplicationLagVeryHigh(tabletHealth *tabletHealth) bool { // The default for this is 2h, same as the discovery_high_replication_lag_minimum_serving here. // * degraded_threshold: this is only used by vttablet for display. It should match // discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. -func FilterStatsByReplicationLag(tabletHealthList []*tabletHealth) []*tabletHealth { +func FilterStatsByReplicationLag(tabletHealthList []*TabletHealth) []*TabletHealth { return filterStatsByLag(tabletHealthList) } -func filterStatsByLag(tabletHealthList []*tabletHealth) []*tabletHealth { +func filterStatsByLag(tabletHealthList []*TabletHealth) []*TabletHealth { list := make([]tabletLagSnapshot, 0, len(tabletHealthList)) // filter non-serving tablets and those with very high replication lag for _, ts := range tabletHealthList { @@ -87,7 +87,7 @@ func filterStatsByLag(tabletHealthList []*tabletHealth) []*tabletHealth { sort.Sort(tabletLagSnapshotList(list)) // Pick those with low replication lag, but at least minNumTablets tablets regardless. - res := make([]*tabletHealth, 0, len(list)) + res := make([]*TabletHealth, 0, len(list)) for i := 0; i < len(list); i++ { if !IsReplicationLagHigh(list[i].ts) || i < *minNumTablets { res = append(res, list[i].ts) @@ -97,7 +97,7 @@ func filterStatsByLag(tabletHealthList []*tabletHealth) []*tabletHealth { } type tabletLagSnapshot struct { - ts *tabletHealth + ts *TabletHealth replag uint32 } type tabletLagSnapshotList []tabletLagSnapshot diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index 77a01c822ed..c920da0cd07 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -31,17 +31,17 @@ func testSetMinNumTablets(newMin int) { func TestFilterByReplicationLagUnhealthy(t *testing.T) { // 1 healthy serving tablet, 1 not healhty - ts1 := &tabletHealth{ + ts1 := &TabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{}, } - ts2 := &tabletHealth{ + ts2 := &TabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: false, Stats: &querypb.RealtimeStats{}, } - got := FilterStatsByReplicationLag([]*tabletHealth{ts1, ts2}) + got := FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2}) if len(got) != 1 { t.Errorf("len(FilterStatsByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}])) = %v, want 1", len(got)) } @@ -102,9 +102,9 @@ func TestFilterByReplicationLag(t *testing.T) { } for _, tc := range cases { - lts := make([]*tabletHealth, len(tc.input)) + lts := make([]*TabletHealth, len(tc.input)) for i, lag := range tc.input { - lts[i] = &tabletHealth{ + lts[i] = &TabletHealth{ Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, @@ -130,52 +130,52 @@ func TestFilterByReplicationLagThreeTabletMin(t *testing.T) { // Use at least 3 tablets if possible testSetMinNumTablets(3) // lags of (1s, 1s, 10m, 11m) - returns at least32 items where the slightly delayed ones that are returned are the 10m and 11m ones. - ts1 := &tabletHealth{ + ts1 := &TabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &tabletHealth{ + ts2 := &TabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts3 := &tabletHealth{ + ts3 := &TabletHealth{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts4 := &tabletHealth{ + ts4 := &TabletHealth{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - got := FilterStatsByReplicationLag([]*tabletHealth{ts1, ts2, ts3, ts4}) + got := FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts1) || !got[1].DeepEqual(ts2) || !got[2].DeepEqual(ts3) { t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } // lags of (11m, 10m, 1s, 1s) - reordered tablets returns the same 3 items where the slightly delayed one that is returned is the 10m and 11m ones. - ts1 = &tabletHealth{ + ts1 = &TabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - ts2 = &tabletHealth{ + ts2 = &TabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts3 = &tabletHealth{ + ts3 = &TabletHealth{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts4 = &tabletHealth{ + ts4 = &TabletHealth{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - got = FilterStatsByReplicationLag([]*tabletHealth{ts1, ts2, ts3, ts4}) + got = FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts3) || !got[1].DeepEqual(ts4) || !got[2].DeepEqual(ts2) { t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } @@ -187,32 +187,32 @@ func TestFilterStatsByReplicationLagOneTabletMin(t *testing.T) { // Use at least 1 tablets if possible testSetMinNumTablets(1) // lags of (1s, 100m) - return only healthy tablet if that is all that is available. - ts1 := &tabletHealth{ + ts1 := &TabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &tabletHealth{ + ts2 := &TabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got := FilterStatsByReplicationLag([]*tabletHealth{ts1, ts2}) + got := FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { t.Errorf("FilterStatsByReplicationLag([1s, 100m]) = %+v, want [1s]", got) } // lags of (1m, 100m) - return only healthy tablet if that is all that is healthy enough. - ts1 = &tabletHealth{ + ts1 = &TabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60}, } - ts2 = &tabletHealth{ + ts2 = &TabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got = FilterStatsByReplicationLag([]*tabletHealth{ts1, ts2}) + got = FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { t.Errorf("FilterStatsByReplicationLag([1m, 100m]) = %+v, want [1m]", got) } diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index df1031036cf..6efbfd91a3a 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -17,13 +17,13 @@ import ( // TabletHealth maintains the health status of a tablet. A map of this // structure is maintained in HealthCheckImpl. -type tabletHealth struct { +type TabletHealth struct { mu sync.Mutex // cancelFunc must be called before discarding TabletHealth. // This will ensure that the associated checkConn goroutine will terminate. cancelFunc context.CancelFunc - // conn is the connection associated with the tablet. - conn queryservice.QueryService + // Conn is the connection associated with the tablet. + Conn queryservice.QueryService // Tablet is the tablet object that was sent to HealthCheck.AddTablet. Tablet *topodata.Tablet // Target is the current target as returned by the streaming @@ -46,17 +46,17 @@ type tabletHealth struct { LastError error } -// String is defined because we want to print a []*tabletHealth array nicely. -func (th *tabletHealth) String() string { +// String is defined because we want to print a []*TabletHealth array nicely. +func (th *TabletHealth) String() string { th.mu.Lock() defer th.mu.Unlock() return fmt.Sprintf("TabletHealth{Tablet: %v,Target: %v,Up: %v,Serving: %v, MasterTermStartTime: %v, Stats: %v, LastError: %v", th.Tablet, th.Target, th.Up, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) } -// DeepEqual compares two tabletHealth. Since we include protos, we +// DeepEqual compares two TabletHealth. Since we include protos, we // need to use proto.Equal on these. -func (th *tabletHealth) DeepEqual(other *tabletHealth) bool { +func (th *TabletHealth) DeepEqual(other *TabletHealth) bool { return proto.Equal(th.Tablet, other.Tablet) && proto.Equal(th.Target, other.Target) && th.Up == other.Up && @@ -68,7 +68,7 @@ func (th *tabletHealth) DeepEqual(other *tabletHealth) bool { } // GetTabletHostPort formats a tablet host port address. -func (th *tabletHealth) GetTabletHostPort() string { +func (th *TabletHealth) GetTabletHostPort() string { th.mu.Lock() hostname := th.Tablet.Hostname vtPort := th.Tablet.PortMap["vt"] @@ -78,7 +78,7 @@ func (th *tabletHealth) GetTabletHostPort() string { // GetHostNameLevel returns the specified hostname level. If the level does not exist it will pick the closest level. // This seems unused but can be utilized by certain url formatting templates. See getTabletDebugURL for more details. -func (th *tabletHealth) GetHostNameLevel(level int) string { +func (th *TabletHealth) GetHostNameLevel(level int) string { th.mu.Lock() hostname := th.Tablet.Hostname th.mu.Unlock() @@ -96,8 +96,8 @@ func (th *tabletHealth) GetHostNameLevel(level int) string { // getTabletDebugURL formats a debug url to the tablet. // It uses a format string that can be passed into the app to format // the debug URL to accommodate different network setups. It applies -// the html/template string defined to a tabletHealth object. The -// format string can refer to members and functions of tabletHealth +// the html/template string defined to a TabletHealth object. The +// format string can refer to members and functions of TabletHealth // like a regular html/template string. // // For instance given a tablet with hostname:port of host.dc.domain:22 @@ -105,20 +105,20 @@ func (th *tabletHealth) GetHostNameLevel(level int) string { // http://{{.GetTabletHostPort}} -> http://host.dc.domain:22 // https://{{.Tablet.Hostname}} -> https://host.dc.domain // https://{{.GetHostNameLevel 0}}.bastion.corp -> https://host.bastion.corp -func (th *tabletHealth) getTabletDebugURL() string { +func (th *TabletHealth) getTabletDebugURL() string { var buffer bytes.Buffer tabletURLTemplate.Execute(&buffer, th) return buffer.String() } -func (th *tabletHealth) deleteConnLocked() { +func (th *TabletHealth) deleteConnLocked() { th.mu.Lock() defer th.mu.Unlock() th.Up = false - th.conn = nil + th.Conn = nil th.cancelFunc() } -func (th *tabletHealth) isHealthy() bool { +func (th *TabletHealth) isHealthy() bool { return th.Serving && th.LastError == nil && th.Stats != nil && !IsReplicationLagVeryHigh(th) } diff --git a/go/vt/discovery/tablets_cache_status.go b/go/vt/discovery/tablets_cache_status.go index 4f7cfcd8987..6b0a1d4efb6 100644 --- a/go/vt/discovery/tablets_cache_status.go +++ b/go/vt/discovery/tablets_cache_status.go @@ -19,7 +19,7 @@ type TabletsCacheStatus struct { } // TabletStatsList is used for sorting. -type TabletStatsList []*tabletHealth +type TabletStatsList []*TabletHealth // Len is part of sort.Interface. func (tsl TabletStatsList) Len() int { diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index 42061f25909..73b10ea15e6 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -18,10 +18,13 @@ package vtgate import ( "fmt" + "math/rand" "sort" "sync" "time" + "vitess.io/vitess/go/vt/topo/topoproto" + "golang.org/x/net/context" "vitess.io/vitess/go/vt/discovery" @@ -163,9 +166,7 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, } } if !match { - return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, - "requested tablet type %v is not part of the allowed tablet types for this vtgate: %+v", - target.TabletType.String(), discovery.AllowedTabletTypes) + return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "requested tablet type %v is not part of the allowed tablet types for this vtgate: %+v", target.TabletType.String(), discovery.AllowedTabletTypes) } } @@ -197,14 +198,38 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, } } - tabletLastUsed, conn, connErr := gw.hc.GetTabletAndConnection(target, gw.localCell, invalidTablets) - // execute - if connErr != nil { - err = connErr + tablets := gw.hc.GetHealthyTabletStats(target) + if len(tablets) == 0 { + // fail fast if there is no tablet + err = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no valid tablet") + break + } + gw.shuffleTablets(gw.localCell, tablets) + + var tabletLastUsed string + var conn queryservice.QueryService + // skip tablets we tried before + for _, t := range tablets { + tabletLastUsed = topoproto.TabletAliasString(t.Tablet.Alias) + if _, ok := invalidTablets[tabletLastUsed]; !ok { + conn = t.Conn + break + } else { + tabletLastUsed = "" + } + } + if tabletLastUsed == "" { + if err == nil { + // do not override error from last attempt. + err = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no available connection") + } break } + + // execute if conn == nil { - err = vterrors.Errorf(vtrpcpb.Code_UNAVAILABLE, "no connection for target %v on attempt #%v", target, i+1) + err = vterrors.Errorf(vtrpcpb.Code_UNAVAILABLE, "no connection for tablet %v", tabletLastUsed) + invalidTablets[tabletLastUsed] = true continue } @@ -242,3 +267,50 @@ func (gw *TabletGateway) getStatsAggregator(target *querypb.Target) *TabletStatu gw.statusAggregators[key] = aggr return aggr } + +func (gw *TabletGateway) shuffleTablets(cell string, tablets []*discovery.TabletHealth) { + sameCell, diffCell, sameCellMax := 0, 0, -1 + length := len(tablets) + + // move all same cell tablets to the front, this is O(n) + for { + sameCellMax = diffCell - 1 + sameCell = gw.nextTablet(cell, tablets, sameCell, length, true) + diffCell = gw.nextTablet(cell, tablets, diffCell, length, false) + // either no more diffs or no more same cells should stop the iteration + if sameCell < 0 || diffCell < 0 { + break + } + + if sameCell < diffCell { + // fast forward the `sameCell` lookup to `diffCell + 1`, `diffCell` unchanged + sameCell = diffCell + 1 + } else { + // sameCell > diffCell, swap needed + tablets[sameCell], tablets[diffCell] = tablets[diffCell], tablets[sameCell] + sameCell++ + diffCell++ + } + } + + //shuffle in same cell tablets + for i := sameCellMax; i > 0; i-- { + swap := rand.Intn(i + 1) + tablets[i], tablets[swap] = tablets[swap], tablets[i] + } + + //shuffle in diff cell tablets + for i, diffCellMin := length-1, sameCellMax+1; i > diffCellMin; i-- { + swap := rand.Intn(i-sameCellMax) + diffCellMin + tablets[i], tablets[swap] = tablets[swap], tablets[i] + } +} + +func (gw *TabletGateway) nextTablet(cell string, tablets []*discovery.TabletHealth, offset, length int, sameCell bool) int { + for ; offset < length; offset++ { + if (tablets[offset].Tablet.Alias.Cell == cell) == sameCell { + return offset + } + } + return -1 +} From 30c6f475f0adf8f9636e5b8d2b0e6ae53c8ab2db Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 28 Apr 2020 17:27:37 -0700 Subject: [PATCH 12/39] healthcheck: Move topoWatch back into TopologyWatcher and use TabletRecorder interface. Move cellsAliases cache into TopologyWatcher Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 215 ++---------------- go/vt/discovery/legacy_healthcheck.go | 8 +- go/vt/discovery/legacy_topology_watcher.go | 56 ++--- go/vt/discovery/topology_watcher.go | 203 ++++++++++++++++- go/vt/vtgate/discoverygateway.go | 2 +- .../tabletserver/txthrottler/tx_throttler.go | 6 +- .../txthrottler/tx_throttler_test.go | 2 +- 7 files changed, 250 insertions(+), 242 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 3db81f4e49f..b193c2b33f1 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -153,14 +153,11 @@ func init() { flag.Var(&KeyspacesToWatch, "keyspaces_to_watch", "Specifies which keyspaces this vtgate should have access to while routing queries or accessing the vschema") } -// HealthCheck defines the interface of health checking module. -// The goal of this object is to maintain a StreamHealth RPC -// to a lot of tablets. Tablets are added / removed by calling the -// AddTablet / RemoveTablet methods (other discovery module objects -// can for instance watch the topology and call these). -type HealthCheck interface { +// TabletRecorder is the part of the HealthCheck interface that can +// add or remove tablets. We define it as a sub-interface here so +// that it can be implemented by other users of TopologyWatcher +type TabletRecorder interface { // AddTablet adds the tablet. - // Name is an alternate name, like an address. AddTablet(tablet *topodatapb.Tablet) // RemoveTablet removes the tablet. @@ -168,6 +165,14 @@ type HealthCheck interface { // ReplaceTablet does an AddTablet and RemoveTablet in one call, effectively replacing the old tablet with the new. ReplaceTablet(old, new *topodatapb.Tablet) +} + +// HealthCheck defines the interface of health checking module. +// The goal of this object is to maintain a StreamHealth RPC +// to a lot of tablets. Tablets are added / removed by calling the +// AddTablet / RemoveTablet methods (other discovery module objects +// can for instance watch the topology and call these). +type HealthCheck interface { // RegisterStats registers the connection counts and checksum stats. // It can only be called on one Healthcheck object per process. RegisterStats() @@ -202,15 +207,10 @@ type HealthCheckImpl struct { // contains a map of TabletHealth keyed by tablet alias for each tablet relevant to the keyspace.shard.tabletType // TODO should we include cell in key? entries map[string]map[string]*TabletHealth - // connsWG keeps track of all launched Go routines that monitor tablet connections. connsWG sync.WaitGroup - // topology watchers that inform healthcheck of tablets being added and deleted topoWatchers []*TopologyWatcher - - // cellAliases is a cache of cell aliases - cellAliases map[string]string } // HealthCheckConn is a structure that lives within the scope of @@ -242,6 +242,12 @@ type healthCheckConn struct { func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string) HealthCheck { log.Infof("loading tablets for cells: %v", *CellsToWatch) + hc := &HealthCheckImpl{ + ts: topoServer, + cell: localCell, + retryDelay: retryDelay, + healthCheckTimeout: healthCheckTimeout, + } var topoWatchers []*TopologyWatcher var filter TabletFilter cells := strings.Split(*CellsToWatch, ",") @@ -265,174 +271,22 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur } else if len(KeyspacesToWatch) > 0 { filter = NewFilterByKeyspace(c, KeyspacesToWatch) } - topoWatchers = append(topoWatchers, NewCellTabletsWatcher(ctx, topoServer, filter, c, *RefreshInterval, *RefreshKnownTablets, *TopoReadConcurrency)) - } - - hc := &HealthCheckImpl{ - ts: topoServer, - cell: localCell, - retryDelay: retryDelay, - healthCheckTimeout: healthCheckTimeout, - cellAliases: make(map[string]string), - topoWatchers: topoWatchers, + topoWatchers = append(topoWatchers, NewCellTabletsWatcher(ctx, topoServer, hc, filter, c, *RefreshInterval, *RefreshKnownTablets, *TopoReadConcurrency)) } + hc.topoWatchers = topoWatchers healthcheckOnce.Do(func() { http.Handle("/debug/gateway", hc) }) // start the topo watches here for _, tw := range hc.topoWatchers { - go hc.watchTopo(tw) + go tw.watchTopo() } return hc } -func (hc *HealthCheckImpl) watchTopo(tw *TopologyWatcher) { - tw.wg.Add(1) - defer tw.wg.Done() - ticker := time.NewTicker(tw.refreshInterval) - defer ticker.Stop() - for { - hc.loadTablets(tw) - select { - case <-tw.ctx.Done(): - return - case <-ticker.C: - } - } -} - -func (hc *HealthCheckImpl) loadTablets(tw *TopologyWatcher) { - var wg sync.WaitGroup - newTablets := make(map[string]*tabletInfo) - replacedTablets := make(map[string]*tabletInfo) - - tabletAliases, err := tw.getTablets(tw) - topologyWatcherOperations.Add(topologyWatcherOpListTablets, 1) - if err != nil { - topologyWatcherErrors.Add(topologyWatcherOpListTablets, 1) - select { - case <-tw.ctx.Done(): - return - default: - } - log.Errorf("cannot get tablets for cell: %v: %v", tw.cell, err) - return - } - - // Accumulate a list of all known alias strings to use later - // when sorting - tabletAliasStrs := make([]string, 0, len(tabletAliases)) - - tw.mu.Lock() - for _, tAlias := range tabletAliases { - aliasStr := topoproto.TabletAliasString(tAlias) - tabletAliasStrs = append(tabletAliasStrs, aliasStr) - - if !tw.refreshKnownTablets { - if val, ok := tw.tablets[aliasStr]; ok { - newTablets[aliasStr] = val - continue - } - } - - wg.Add(1) - go func(alias *topodatapb.TabletAlias) { - defer wg.Done() - tw.sem <- 1 // Wait for active queue to drain. - tablet, err := tw.topoServer.GetTablet(tw.ctx, alias) - topologyWatcherOperations.Add(topologyWatcherOpGetTablet, 1) - <-tw.sem // Done; enable next request to run - if err != nil { - topologyWatcherErrors.Add(topologyWatcherOpGetTablet, 1) - select { - case <-tw.ctx.Done(): - return - default: - } - log.Errorf("cannot get tablet for alias %v: %v", alias, err) - return - } - if !(hc.isTabletInCell(tablet.Tablet) && (tw.tabletFilter == nil || tw.tabletFilter.IsIncluded(tablet.Tablet))) { - return - } - tw.mu.Lock() - aliasStr := topoproto.TabletAliasString(alias) - newTablets[aliasStr] = &tabletInfo{ - alias: aliasStr, - key: TabletToMapKey(tablet.Tablet), - tablet: tablet.Tablet, - } - tw.mu.Unlock() - }(tAlias) - } - - tw.mu.Unlock() - wg.Wait() - tw.mu.Lock() - - for alias, newVal := range newTablets { - if val, ok := tw.tablets[alias]; !ok { - // Check if there's a tablet with the same address key but a - // different alias. If so, replace it and keep track of the - // replaced alias to make sure it isn't removed later. - found := false - for _, otherVal := range tw.tablets { - if newVal.key == otherVal.key { - found = true - hc.ReplaceTablet(otherVal.tablet, newVal.tablet) - topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) - replacedTablets[otherVal.alias] = newVal - } - } - if !found { - hc.AddTablet(newVal.tablet) - topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1) - } - - } else if val.key != newVal.key { - // Handle the case where the same tablet alias is now reporting - // a different address key. - replacedTablets[alias] = newVal - hc.ReplaceTablet(val.tablet, newVal.tablet) - topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) - } - } - - for _, val := range tw.tablets { - if _, ok := newTablets[val.alias]; !ok { - if _, ok2 := replacedTablets[val.alias]; !ok2 { - hc.RemoveTablet(val.tablet) - topologyWatcherOperations.Add(topologyWatcherOpRemoveTablet, 1) - } - } - } - tw.tablets = newTablets - if !tw.firstLoadDone { - tw.firstLoadDone = true - close(tw.firstLoadChan) - } - - // iterate through the tablets in a stable order and compute a - // checksum of the tablet map - sort.Strings(tabletAliasStrs) - var buf bytes.Buffer - for _, alias := range tabletAliasStrs { - tabletInfo, ok := tw.tablets[alias] - if ok { - buf.WriteString(alias) - buf.WriteString(tabletInfo.key) - } - } - tw.topoChecksum = crc32.ChecksumIEEE(buf.Bytes()) - tw.lastRefresh = time.Now() - - tw.mu.Unlock() - -} - // RegisterStats registers the connection counts stats func (hc *HealthCheckImpl) RegisterStats() { stats.NewGaugeDurationFunc( @@ -940,33 +794,6 @@ func (hc *HealthCheckImpl) getTabletStats(target *querypb.Target) []*TabletHealt return result } -func (hc *HealthCheckImpl) getAliasByCell(cell string) string { - hc.mu.Lock() - defer hc.mu.Unlock() - - if alias, ok := hc.cellAliases[cell]; ok { - return alias - } - - alias := topo.GetAliasByCell(context.Background(), hc.ts, cell) - hc.cellAliases[cell] = alias - - return alias -} - -func (hc *HealthCheckImpl) isTabletInCell(tablet *topodatapb.Tablet) bool { - if tablet.Type == topodatapb.TabletType_MASTER { - return true - } - if tablet.Alias.Cell == hc.cell { - return true - } - if hc.getAliasByCell(tablet.Alias.Cell) == hc.getAliasByCell(hc.cell) { - return true - } - return false -} - // WaitForTablets waits for at least one tablet in the given // keyspace / shard / tablet type before returning. The tablets do not // have to be healthy. It will return ctx.Err() if the context is canceled. diff --git a/go/vt/discovery/legacy_healthcheck.go b/go/vt/discovery/legacy_healthcheck.go index 678fc1f0e2f..7e17734ad8c 100644 --- a/go/vt/discovery/legacy_healthcheck.go +++ b/go/vt/discovery/legacy_healthcheck.go @@ -251,10 +251,10 @@ func (e *LegacyTabletStats) TrivialStatsUpdate(n *LegacyTabletStats) bool { return false } -// TabletRecorder is the part of the LegacyHealthCheck interface that can +// LegacyTabletRecorder is the part of the LegacyHealthCheck interface that can // add or remove tablets. We define it as a sub-interface here so we // can add filters on tablets if needed. -type TabletRecorder interface { +type LegacyTabletRecorder interface { // AddTablet adds the tablet. // Name is an alternate name, like an address. AddTablet(tablet *topodatapb.Tablet, name string) @@ -278,10 +278,10 @@ type TabletRecorder interface { // below and pass in the Key string which is also sent to the // listener in each update (as it is part of LegacyTabletStats). type LegacyHealthCheck interface { - // TabletRecorder interface adds AddTablet and RemoveTablet methods. + // LegacyTabletRecorder interface adds AddTablet and RemoveTablet methods. // AddTablet adds the tablet, and starts health check on it. // RemoveTablet removes the tablet, and stops its StreamHealth RPC. - TabletRecorder + LegacyTabletRecorder // RegisterStats registers the connection counts and checksum stats. // It can only be called on one Healthcheck object per process. diff --git a/go/vt/discovery/legacy_topology_watcher.go b/go/vt/discovery/legacy_topology_watcher.go index b2b407987e3..b00e151c698 100644 --- a/go/vt/discovery/legacy_topology_watcher.go +++ b/go/vt/discovery/legacy_topology_watcher.go @@ -38,7 +38,7 @@ import ( // NewLegacyCellTabletsWatcher returns a LegacyTopologyWatcher that monitors all // the tablets in a cell, and starts refreshing. -func NewLegacyCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *LegacyTopologyWatcher { +func NewLegacyCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, tr LegacyTabletRecorder, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *LegacyTopologyWatcher { return NewLegacyTopologyWatcher(ctx, topoServer, tr, cell, refreshInterval, refreshKnownTablets, topoReadConcurrency, func(tw *LegacyTopologyWatcher) ([]*topodatapb.TabletAlias, error) { return tw.topoServer.GetTabletsByCell(ctx, tw.cell) }) @@ -46,7 +46,7 @@ func NewLegacyCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, t // NewLegacyShardReplicationWatcher returns a LegacyTopologyWatcher that // monitors the tablets in a cell/keyspace/shard, and starts refreshing. -func NewLegacyShardReplicationWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) *LegacyTopologyWatcher { +func NewLegacyShardReplicationWatcher(ctx context.Context, topoServer *topo.Server, tr LegacyTabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) *LegacyTopologyWatcher { return NewLegacyTopologyWatcher(ctx, topoServer, tr, cell, refreshInterval, true /* RefreshKnownTablets */, topoReadConcurrency, func(tw *LegacyTopologyWatcher) ([]*topodatapb.TabletAlias, error) { sri, err := tw.topoServer.GetShardReplication(ctx, tw.cell, keyspace, shard) switch { @@ -69,11 +69,11 @@ func NewLegacyShardReplicationWatcher(ctx context.Context, topoServer *topo.Serv // LegacyTopologyWatcher polls tablet from a configurable set of tablets // periodically. When tablets are added / removed, it calls -// the TabletRecorder AddTablet / RemoveTablet interface appropriately. +// the LegacyTabletRecorder AddTablet / RemoveTablet interface appropriately. type LegacyTopologyWatcher struct { // set at construction time topoServer *topo.Server - tr TabletRecorder + tr LegacyTabletRecorder cell string refreshInterval time.Duration refreshKnownTablets bool @@ -100,7 +100,7 @@ type LegacyTopologyWatcher struct { // NewLegacyTopologyWatcher returns a LegacyTopologyWatcher that monitors all // the tablets in a cell, and starts refreshing. -func NewLegacyTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *LegacyTopologyWatcher) ([]*topodatapb.TabletAlias, error)) *LegacyTopologyWatcher { +func NewLegacyTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr LegacyTabletRecorder, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *LegacyTopologyWatcher) ([]*topodatapb.TabletAlias, error)) *LegacyTopologyWatcher { tw := &LegacyTopologyWatcher{ topoServer: topoServer, tr: tr, @@ -121,7 +121,7 @@ func NewLegacyTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr T return tw } -// watch polls all tablets and notifies TabletRecorder by adding/removing tablets. +// watch polls all tablets and notifies LegacyTabletRecorder by adding/removing tablets. func (tw *LegacyTopologyWatcher) watch() { defer tw.wg.Done() ticker := time.NewTicker(tw.refreshInterval) @@ -136,7 +136,7 @@ func (tw *LegacyTopologyWatcher) watch() { } } -// loadTablets reads all tablets from topology, and updates TabletRecorder. +// loadTablets reads all tablets from topology, and updates LegacyTabletRecorder. func (tw *LegacyTopologyWatcher) loadTablets() { var wg sync.WaitGroup newTablets := make(map[string]*tabletInfo) @@ -263,7 +263,7 @@ func (tw *LegacyTopologyWatcher) loadTablets() { } // WaitForInitialTopology waits until the watcher reads all of the topology data -// for the first time and transfers the information to TabletRecorder via its +// for the first time and transfers the information to LegacyTabletRecorder via its // AddTablet() method. func (tw *LegacyTopologyWatcher) WaitForInitialTopology() error { select { @@ -274,7 +274,7 @@ func (tw *LegacyTopologyWatcher) WaitForInitialTopology() error { } } -// Stop stops the watcher. It does not clean up the tablets added to TabletRecorder. +// Stop stops the watcher. It does not clean up the tablets added to LegacyTabletRecorder. func (tw *LegacyTopologyWatcher) Stop() { tw.cancelFunc() // wait for watch goroutine to finish. @@ -297,22 +297,22 @@ func (tw *LegacyTopologyWatcher) TopoChecksum() uint32 { return tw.topoChecksum } -// LegacyFilterByShard is a TabletRecorder filter that filters tablets by +// LegacyFilterByShard is a LegacyTabletRecorder filter that filters tablets by // keyspace/shard. type LegacyFilterByShard struct { - // tr is the underlying TabletRecorder to forward requests too - tr TabletRecorder + // tr is the underlying LegacyTabletRecorder to forward requests too + tr LegacyTabletRecorder // filters is a map of keyspace to filters for shards filters map[string][]*filterShard } // NewLegacyFilterByShard creates a new LegacyFilterByShard on top of an existing -// TabletRecorder. Each filter is a keyspace|shard entry, where shard +// LegacyTabletRecorder. Each filter is a keyspace|shard entry, where shard // can either be a shard name, or a keyrange. All tablets that match // at least one keyspace|shard tuple will be forwarded to the -// underlying TabletRecorder. -func NewLegacyFilterByShard(tr TabletRecorder, filters []string) (*LegacyFilterByShard, error) { +// underlying LegacyTabletRecorder. +func NewLegacyFilterByShard(tr LegacyTabletRecorder, filters []string) (*LegacyFilterByShard, error) { m := make(map[string][]*filterShard) for _, filter := range filters { parts := strings.Split(filter, "|") @@ -349,21 +349,21 @@ func NewLegacyFilterByShard(tr TabletRecorder, filters []string) (*LegacyFilterB }, nil } -// AddTablet is part of the TabletRecorder interface. +// AddTablet is part of the LegacyTabletRecorder interface. func (fbs *LegacyFilterByShard) AddTablet(tablet *topodatapb.Tablet, name string) { if fbs.isIncluded(tablet) { fbs.tr.AddTablet(tablet, name) } } -// RemoveTablet is part of the TabletRecorder interface. +// RemoveTablet is part of the LegacyTabletRecorder interface. func (fbs *LegacyFilterByShard) RemoveTablet(tablet *topodatapb.Tablet) { if fbs.isIncluded(tablet) { fbs.tr.RemoveTablet(tablet) } } -// ReplaceTablet is part of the TabletRecorder interface. +// ReplaceTablet is part of the LegacyTabletRecorder interface. func (fbs *LegacyFilterByShard) ReplaceTablet(old, new *topodatapb.Tablet, name string) { if fbs.isIncluded(old) && fbs.isIncluded(new) { fbs.tr.ReplaceTablet(old, new, name) @@ -371,7 +371,7 @@ func (fbs *LegacyFilterByShard) ReplaceTablet(old, new *topodatapb.Tablet, name } // isIncluded returns true iff the tablet's keyspace and shard should be -// forwarded to the underlying TabletRecorder. +// forwarded to the underlying LegacyTabletRecorder. func (fbs *LegacyFilterByShard) isIncluded(tablet *topodatapb.Tablet) bool { canonical, kr, err := topo.ValidateShardName(tablet.Shard) if err != nil { @@ -392,18 +392,18 @@ func (fbs *LegacyFilterByShard) isIncluded(tablet *topodatapb.Tablet) bool { return false } -// LegacyFilterByKeyspace is a TabletRecorder filter that filters tablets by +// LegacyFilterByKeyspace is a LegacyTabletRecorder filter that filters tablets by // keyspace type LegacyFilterByKeyspace struct { - tr TabletRecorder + tr LegacyTabletRecorder keyspaces map[string]bool } // NewLegacyFilterByKeyspace creates a new LegacyFilterByKeyspace on top of an existing -// TabletRecorder. Each filter is a keyspace entry. All tablets that match -// a keyspace will be forwarded to the underlying TabletRecorder. -func NewLegacyFilterByKeyspace(tr TabletRecorder, selectedKeyspaces []string) *LegacyFilterByKeyspace { +// LegacyTabletRecorder. Each filter is a keyspace entry. All tablets that match +// a keyspace will be forwarded to the underlying LegacyTabletRecorder. +func NewLegacyFilterByKeyspace(tr LegacyTabletRecorder, selectedKeyspaces []string) *LegacyFilterByKeyspace { m := make(map[string]bool) for _, keyspace := range selectedKeyspaces { m[keyspace] = true @@ -415,21 +415,21 @@ func NewLegacyFilterByKeyspace(tr TabletRecorder, selectedKeyspaces []string) *L } } -// AddTablet is part of the TabletRecorder interface. +// AddTablet is part of the LegacyTabletRecorder interface. func (fbk *LegacyFilterByKeyspace) AddTablet(tablet *topodatapb.Tablet, name string) { if fbk.isIncluded(tablet) { fbk.tr.AddTablet(tablet, name) } } -// RemoveTablet is part of the TabletRecorder interface. +// RemoveTablet is part of the LegacyTabletRecorder interface. func (fbk *LegacyFilterByKeyspace) RemoveTablet(tablet *topodatapb.Tablet) { if fbk.isIncluded(tablet) { fbk.tr.RemoveTablet(tablet) } } -// ReplaceTablet is part of the TabletRecorder interface. +// ReplaceTablet is part of the LegacyTabletRecorder interface. func (fbk *LegacyFilterByKeyspace) ReplaceTablet(old *topodatapb.Tablet, new *topodatapb.Tablet, name string) { if old.Keyspace != new.Keyspace { log.Errorf("Error replacing old tablet in %v with new tablet in %v", old.Keyspace, new.Keyspace) @@ -442,7 +442,7 @@ func (fbk *LegacyFilterByKeyspace) ReplaceTablet(old *topodatapb.Tablet, new *to } // isIncluded returns true if the tablet's keyspace should be -// forwarded to the underlying TabletRecorder. +// forwarded to the underlying LegacyTabletRecorder. func (fbk *LegacyFilterByKeyspace) isIncluded(tablet *topodatapb.Tablet) bool { _, exist := fbk.keyspaces[tablet.Keyspace] return exist diff --git a/go/vt/discovery/topology_watcher.go b/go/vt/discovery/topology_watcher.go index c4c7adf9c79..d9454a8158e 100644 --- a/go/vt/discovery/topology_watcher.go +++ b/go/vt/discovery/topology_watcher.go @@ -17,11 +17,16 @@ limitations under the License. package discovery import ( + "bytes" "fmt" + "hash/crc32" + "sort" "strings" "sync" "time" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/key" "golang.org/x/net/context" @@ -57,18 +62,19 @@ type tabletInfo struct { // NewCellTabletsWatcher returns a TopologyWatcher that monitors all // the tablets in a cell, and starts refreshing. -func NewCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, f TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *TopologyWatcher { - return NewTopologyWatcher(ctx, topoServer, f, cell, refreshInterval, refreshKnownTablets, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) { +func NewCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, f TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *TopologyWatcher { + return NewTopologyWatcher(ctx, topoServer, tr, f, cell, refreshInterval, refreshKnownTablets, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) { return tw.topoServer.GetTabletsByCell(ctx, tw.cell) }) } // TopologyWatcher polls tablet from a configurable set of tablets // periodically. When tablets are added / removed, it calls -// the TabletRecorder AddTablet / RemoveTablet interface appropriately. +// the LegacyTabletRecorder AddTablet / RemoveTablet interface appropriately. type TopologyWatcher struct { // set at construction time topoServer *topo.Server + tabletRecorder TabletRecorder tabletFilter TabletFilter cell string refreshInterval time.Duration @@ -92,13 +98,16 @@ type TopologyWatcher struct { firstLoadDone bool // firstLoadChan is closed when the initial loading of topology data is done. firstLoadChan chan struct{} + // cellAliases is a cache of cell aliases + cellAliases map[string]string } // NewTopologyWatcher returns a TopologyWatcher that monitors all // the tablets in a cell, and starts refreshing. -func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, filter TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error)) *TopologyWatcher { +func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, filter TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error)) *TopologyWatcher { tw := &TopologyWatcher{ topoServer: topoServer, + tabletRecorder: tr, tabletFilter: filter, cell: cell, refreshInterval: refreshInterval, @@ -106,6 +115,7 @@ func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, filter Tab getTablets: getTablets, sem: make(chan int, topoReadConcurrency), tablets: make(map[string]*tabletInfo), + cellAliases: make(map[string]string), } tw.firstLoadChan = make(chan struct{}) @@ -116,7 +126,7 @@ func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, filter Tab } // WaitForInitialTopology waits until the watcher reads all of the topology data -// for the first time and transfers the information to TabletRecorder via its +// for the first time and transfers the information to LegacyTabletRecorder via its // AddTablet() method. func (tw *TopologyWatcher) WaitForInitialTopology() error { select { @@ -127,7 +137,7 @@ func (tw *TopologyWatcher) WaitForInitialTopology() error { } } -// Stop stops the watcher. It does not clean up the tablets added to TabletRecorder. +// Stop stops the watcher. It does not clean up the tablets added to LegacyTabletRecorder. func (tw *TopologyWatcher) Stop() { tw.cancelFunc() // wait for watch goroutine to finish. @@ -150,6 +160,177 @@ func (tw *TopologyWatcher) TopoChecksum() uint32 { return tw.topoChecksum } +func (tw *TopologyWatcher) watchTopo() { + tw.wg.Add(1) + defer tw.wg.Done() + ticker := time.NewTicker(tw.refreshInterval) + defer ticker.Stop() + for { + tw.loadTablets() + select { + case <-tw.ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (tw *TopologyWatcher) loadTablets() { + var wg sync.WaitGroup + newTablets := make(map[string]*tabletInfo) + replacedTablets := make(map[string]*tabletInfo) + + tabletAliases, err := tw.getTablets(tw) + topologyWatcherOperations.Add(topologyWatcherOpListTablets, 1) + if err != nil { + topologyWatcherErrors.Add(topologyWatcherOpListTablets, 1) + select { + case <-tw.ctx.Done(): + return + default: + } + log.Errorf("cannot get tablets for cell: %v: %v", tw.cell, err) + return + } + + // Accumulate a list of all known alias strings to use later + // when sorting + tabletAliasStrs := make([]string, 0, len(tabletAliases)) + + tw.mu.Lock() + for _, tAlias := range tabletAliases { + aliasStr := topoproto.TabletAliasString(tAlias) + tabletAliasStrs = append(tabletAliasStrs, aliasStr) + + if !tw.refreshKnownTablets { + if val, ok := tw.tablets[aliasStr]; ok { + newTablets[aliasStr] = val + continue + } + } + + wg.Add(1) + go func(alias *topodatapb.TabletAlias) { + defer wg.Done() + tw.sem <- 1 // Wait for active queue to drain. + tablet, err := tw.topoServer.GetTablet(tw.ctx, alias) + topologyWatcherOperations.Add(topologyWatcherOpGetTablet, 1) + <-tw.sem // Done; enable next request to run + if err != nil { + topologyWatcherErrors.Add(topologyWatcherOpGetTablet, 1) + select { + case <-tw.ctx.Done(): + return + default: + } + log.Errorf("cannot get tablet for alias %v: %v", alias, err) + return + } + if !(tw.isTabletInCell(tablet.Tablet) && (tw.tabletFilter == nil || tw.tabletFilter.IsIncluded(tablet.Tablet))) { + return + } + tw.mu.Lock() + aliasStr := topoproto.TabletAliasString(alias) + newTablets[aliasStr] = &tabletInfo{ + alias: aliasStr, + key: TabletToMapKey(tablet.Tablet), + tablet: tablet.Tablet, + } + tw.mu.Unlock() + }(tAlias) + } + + tw.mu.Unlock() + wg.Wait() + tw.mu.Lock() + + for alias, newVal := range newTablets { + if val, ok := tw.tablets[alias]; !ok { + // Check if there's a tablet with the same address key but a + // different alias. If so, replace it and keep track of the + // replaced alias to make sure it isn't removed later. + found := false + for _, otherVal := range tw.tablets { + if newVal.key == otherVal.key { + found = true + tw.tabletRecorder.ReplaceTablet(otherVal.tablet, newVal.tablet) + topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) + replacedTablets[otherVal.alias] = newVal + } + } + if !found { + tw.tabletRecorder.AddTablet(newVal.tablet) + topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1) + } + + } else if val.key != newVal.key { + // Handle the case where the same tablet alias is now reporting + // a different address key. + replacedTablets[alias] = newVal + tw.tabletRecorder.ReplaceTablet(val.tablet, newVal.tablet) + topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) + } + } + + for _, val := range tw.tablets { + if _, ok := newTablets[val.alias]; !ok { + if _, ok2 := replacedTablets[val.alias]; !ok2 { + tw.tabletRecorder.RemoveTablet(val.tablet) + topologyWatcherOperations.Add(topologyWatcherOpRemoveTablet, 1) + } + } + } + tw.tablets = newTablets + if !tw.firstLoadDone { + tw.firstLoadDone = true + close(tw.firstLoadChan) + } + + // iterate through the tablets in a stable order and compute a + // checksum of the tablet map + sort.Strings(tabletAliasStrs) + var buf bytes.Buffer + for _, alias := range tabletAliasStrs { + tabletInfo, ok := tw.tablets[alias] + if ok { + buf.WriteString(alias) + buf.WriteString(tabletInfo.key) + } + } + tw.topoChecksum = crc32.ChecksumIEEE(buf.Bytes()) + tw.lastRefresh = time.Now() + + tw.mu.Unlock() + +} + +func (tw *TopologyWatcher) getAliasByCell(cell string) string { + tw.mu.Lock() + defer tw.mu.Unlock() + + if alias, ok := tw.cellAliases[cell]; ok { + return alias + } + + alias := topo.GetAliasByCell(context.Background(), tw.topoServer, cell) + tw.cellAliases[cell] = alias + + return alias +} + +func (tw *TopologyWatcher) isTabletInCell(tablet *topodatapb.Tablet) bool { + if tablet.Type == topodatapb.TabletType_MASTER { + return true + } + if tablet.Alias.Cell == tw.cell { + return true + } + if tw.getAliasByCell(tablet.Alias.Cell) == tw.getAliasByCell(tw.cell) { + return true + } + return false +} + // TabletFilter is an interface that can be given to a TopologyWatcher // to be applied as an additional filter on the list of tablets returned by its getTablets function type TabletFilter interface { @@ -173,10 +354,10 @@ type filterShard struct { } // NewFilterByShard creates a new FilterByShard on top of an existing -// TabletRecorder. Each filter is a keyspace|shard entry, where shard +// LegacyTabletRecorder. Each filter is a keyspace|shard entry, where shard // can either be a shard name, or a keyrange. All tablets that match // at least one keyspace|shard tuple will be forwarded to the -// underlying TabletRecorder. +// underlying LegacyTabletRecorder. func NewFilterByShard(filters []string) (*FilterByShard, error) { m := make(map[string][]*filterShard) for _, filter := range filters { @@ -214,7 +395,7 @@ func NewFilterByShard(filters []string) (*FilterByShard, error) { } // IsIncluded returns true iff the tablet's keyspace and shard should be -// forwarded to the underlying TabletRecorder. +// forwarded to the underlying LegacyTabletRecorder. func (fbs *FilterByShard) IsIncluded(tablet *topodatapb.Tablet) bool { canonical, kr, err := topo.ValidateShardName(tablet.Shard) if err != nil { @@ -243,7 +424,7 @@ type FilterByKeyspace struct { // NewFilterByKeyspace creates a new FilterByKeyspace. // Each filter is a keyspace entry. All tablets that match -// a keyspace will be forwarded to the underlying TabletRecorder. +// a keyspace will be forwarded to the underlying LegacyTabletRecorder. func NewFilterByKeyspace(cell string, selectedKeyspaces []string) *FilterByKeyspace { m := make(map[string]bool) for _, keyspace := range selectedKeyspaces { @@ -256,7 +437,7 @@ func NewFilterByKeyspace(cell string, selectedKeyspaces []string) *FilterByKeysp } // IsIncluded returns true if the tablet's keyspace should be -// forwarded to the underlying TabletRecorder. +// forwarded to the underlying LegacyTabletRecorder. func (fbk *FilterByKeyspace) IsIncluded(tablet *topodatapb.Tablet) bool { _, exist := fbk.keyspaces[tablet.Keyspace] return exist diff --git a/go/vt/vtgate/discoverygateway.go b/go/vt/vtgate/discoverygateway.go index b78eb710cbf..8b03db138e1 100644 --- a/go/vt/vtgate/discoverygateway.go +++ b/go/vt/vtgate/discoverygateway.go @@ -114,7 +114,7 @@ func NewDiscoveryGateway(ctx context.Context, hc discovery.LegacyHealthCheck, se if c == "" { continue } - var recorder discovery.TabletRecorder = dg.hc + var recorder discovery.LegacyTabletRecorder = dg.hc if len(discovery.TabletFilters) > 0 { if len(discovery.KeyspacesToWatch) > 0 { log.Exitf("Only one of -keyspaces_to_watch and -tablet_filters may be specified at a time") diff --git a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go index 8900d9cfd9d..001435e92f5 100644 --- a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go +++ b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go @@ -166,7 +166,7 @@ type txThrottlerState struct { // topology watchers and go/vt/throttler. These are provided here so that they can be overridden // in tests to generate mocks. type healthCheckFactoryFunc func() discovery.LegacyHealthCheck -type topologyWatcherFactoryFunc func(topoServer *topo.Server, tr discovery.TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) TopologyWatcherInterface +type topologyWatcherFactoryFunc func(topoServer *topo.Server, tr discovery.LegacyTabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) TopologyWatcherInterface type throttlerFactoryFunc func(name, unit string, threadCount int, maxRate, maxReplicationLag int64) (ThrottlerInterface, error) var ( @@ -181,7 +181,7 @@ func init() { func resetTxThrottlerFactories() { healthCheckFactory = discovery.NewLegacyDefaultHealthCheck - topologyWatcherFactory = func(topoServer *topo.Server, tr discovery.TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) TopologyWatcherInterface { + topologyWatcherFactory = func(topoServer *topo.Server, tr discovery.LegacyTabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) TopologyWatcherInterface { return discovery.NewLegacyShardReplicationWatcher(context.Background(), topoServer, tr, cell, keyspace, shard, refreshInterval, topoReadConcurrency) } throttlerFactory = func(name, unit string, threadCount int, maxRate, maxReplicationLag int64) (ThrottlerInterface, error) { @@ -278,7 +278,7 @@ func newTxThrottlerState(config *txThrottlerConfig, keyspace, shard string, result.topologyWatchers, topologyWatcherFactory( config.topoServer, - result.healthCheck, /* TabletRecorder */ + result.healthCheck, /* LegacyTabletRecorder */ cell, keyspace, shard, diff --git a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go index eb11c553c40..6fea2c65379 100644 --- a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go +++ b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go @@ -66,7 +66,7 @@ func TestEnabledThrottler(t *testing.T) { hcCall2.After(hcCall1) healthCheckFactory = func() discovery.LegacyHealthCheck { return mockHealthCheck } - topologyWatcherFactory = func(topoServer *topo.Server, tr discovery.TabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) TopologyWatcherInterface { + topologyWatcherFactory = func(topoServer *topo.Server, tr discovery.LegacyTabletRecorder, cell, keyspace, shard string, refreshInterval time.Duration, topoReadConcurrency int) TopologyWatcherInterface { if ts != topoServer { t.Errorf("want: %v, got: %v", ts, topoServer) } From b0ce1c5c66adab323c9dae65c47317cb2914462e Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 28 Apr 2020 17:36:38 -0700 Subject: [PATCH 13/39] healthcheck: remove references to TabletToMapKey in new topo watcher Signed-off-by: deepthi --- go/vt/discovery/legacy_topology_watcher.go | 17 ++++++++++++----- go/vt/discovery/topology_watcher.go | 9 +++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/go/vt/discovery/legacy_topology_watcher.go b/go/vt/discovery/legacy_topology_watcher.go index b00e151c698..02370a511a8 100644 --- a/go/vt/discovery/legacy_topology_watcher.go +++ b/go/vt/discovery/legacy_topology_watcher.go @@ -36,6 +36,13 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) +// tabletInfo is used internally by the TopologyWatcher class +type legacyTabletInfo struct { + alias string + key string + tablet *topodatapb.Tablet +} + // NewLegacyCellTabletsWatcher returns a LegacyTopologyWatcher that monitors all // the tablets in a cell, and starts refreshing. func NewLegacyCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, tr LegacyTabletRecorder, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *LegacyTopologyWatcher { @@ -87,7 +94,7 @@ type LegacyTopologyWatcher struct { // mu protects all variables below mu sync.Mutex // tablets contains a map of alias -> tabletInfo for all known tablets - tablets map[string]*tabletInfo + tablets map[string]*legacyTabletInfo // topoChecksum stores a crc32 of the tablets map and is exported as a metric topoChecksum uint32 // lastRefresh records the timestamp of the last topo refresh @@ -109,7 +116,7 @@ func NewLegacyTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr L refreshKnownTablets: refreshKnownTablets, getTablets: getTablets, sem: make(chan int, topoReadConcurrency), - tablets: make(map[string]*tabletInfo), + tablets: make(map[string]*legacyTabletInfo), } tw.firstLoadChan = make(chan struct{}) @@ -139,8 +146,8 @@ func (tw *LegacyTopologyWatcher) watch() { // loadTablets reads all tablets from topology, and updates LegacyTabletRecorder. func (tw *LegacyTopologyWatcher) loadTablets() { var wg sync.WaitGroup - newTablets := make(map[string]*tabletInfo) - replacedTablets := make(map[string]*tabletInfo) + newTablets := make(map[string]*legacyTabletInfo) + replacedTablets := make(map[string]*legacyTabletInfo) tabletAliases, err := tw.getTablets(tw) topologyWatcherOperations.Add(topologyWatcherOpListTablets, 1) @@ -190,7 +197,7 @@ func (tw *LegacyTopologyWatcher) loadTablets() { } tw.mu.Lock() aliasStr := topoproto.TabletAliasString(alias) - newTablets[aliasStr] = &tabletInfo{ + newTablets[aliasStr] = &legacyTabletInfo{ alias: aliasStr, key: TabletToMapKey(tablet.Tablet), tablet: tablet.Tablet, diff --git a/go/vt/discovery/topology_watcher.go b/go/vt/discovery/topology_watcher.go index d9454a8158e..396389c21b9 100644 --- a/go/vt/discovery/topology_watcher.go +++ b/go/vt/discovery/topology_watcher.go @@ -56,7 +56,6 @@ var ( // tabletInfo is used internally by the TopologyWatcher class type tabletInfo struct { alias string - key string tablet *topodatapb.Tablet } @@ -233,7 +232,6 @@ func (tw *TopologyWatcher) loadTablets() { aliasStr := topoproto.TabletAliasString(alias) newTablets[aliasStr] = &tabletInfo{ alias: aliasStr, - key: TabletToMapKey(tablet.Tablet), tablet: tablet.Tablet, } tw.mu.Unlock() @@ -251,7 +249,7 @@ func (tw *TopologyWatcher) loadTablets() { // replaced alias to make sure it isn't removed later. found := false for _, otherVal := range tw.tablets { - if newVal.key == otherVal.key { + if newVal.alias == otherVal.alias { found = true tw.tabletRecorder.ReplaceTablet(otherVal.tablet, newVal.tablet) topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) @@ -263,7 +261,7 @@ func (tw *TopologyWatcher) loadTablets() { topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1) } - } else if val.key != newVal.key { + } else if val.alias != newVal.alias { // Handle the case where the same tablet alias is now reporting // a different address key. replacedTablets[alias] = newVal @@ -291,10 +289,9 @@ func (tw *TopologyWatcher) loadTablets() { sort.Strings(tabletAliasStrs) var buf bytes.Buffer for _, alias := range tabletAliasStrs { - tabletInfo, ok := tw.tablets[alias] + _, ok := tw.tablets[alias] if ok { buf.WriteString(alias) - buf.WriteString(tabletInfo.key) } } tw.topoChecksum = crc32.ChecksumIEEE(buf.Bytes()) From 1abc4f432dac41fd6ad1b8db1340019e6a4cff9d Mon Sep 17 00:00:00 2001 From: deepthi Date: Wed, 29 Apr 2020 12:40:24 -0700 Subject: [PATCH 14/39] healthcheck: notify buffer when failover ends, rename entries -> healthData Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 40 ++++++++++++++++---------------- go/vt/discovery/tablet_health.go | 22 ++++++++++++++++++ go/vt/vtgate/buffer/buffer.go | 35 ++++++++++++++++++++++++++++ go/vt/vtgate/tabletgateway.go | 26 ++++++++++----------- 4 files changed, 89 insertions(+), 34 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index b193c2b33f1..f148ad9e217 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -205,8 +205,7 @@ type HealthCheckImpl struct { // a map keyed by keyspace.shard.tabletType // contains a map of TabletHealth keyed by tablet alias for each tablet relevant to the keyspace.shard.tabletType - // TODO should we include cell in key? - entries map[string]map[string]*TabletHealth + healthData map[string]map[string]*TabletHealth // connsWG keeps track of all launched Go routines that monitor tablet connections. connsWG sync.WaitGroup // topology watchers that inform healthcheck of tablets being added and deleted @@ -333,7 +332,7 @@ func (hc *HealthCheckImpl) servingConnStats() map[string]int64 { res := make(map[string]int64) hc.mu.Lock() defer hc.mu.Unlock() - for key, ths := range hc.entries { + for key, ths := range hc.healthData { for _, th := range ths { if !th.Up || !th.Serving || th.LastError != nil { continue @@ -552,8 +551,8 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St newTargetKey := hc.keyFromTarget(shr.Target) tabletAlias := topoproto.TabletAliasString(currentTablet.Alias) hc.mu.Lock() - delete(hc.entries[oldTargetKey], tabletAlias) - hc.entries[newTargetKey][tabletAlias] = hcc.tabletHealth + delete(hc.healthData[oldTargetKey], tabletAlias) + hc.healthData[newTargetKey][tabletAlias] = hcc.tabletHealth hc.mu.Unlock() } @@ -580,7 +579,7 @@ func (hc *HealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { key := hc.keyFromTablet(tablet) tabletAlias := topoproto.TabletAliasString(tablet.Alias) - ths, ok := hc.entries[key] + ths, ok := hc.healthData[key] if !ok { log.Warningf("Something is wrong, we have no health data for tablet: %v's target: %v", tabletAlias, key) return @@ -599,7 +598,7 @@ func (hc *HealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { // name is an optional tag for the tablet, e.g. an alternative address. func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet) { hc.mu.Lock() - if hc.entries == nil { + if hc.healthData == nil { // already closed. hc.mu.Unlock() return @@ -623,9 +622,9 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet) { // add to our datastore key := hc.keyFromTarget(target) tabletAlias := topoproto.TabletAliasString(tablet.Alias) - if ths, ok := hc.entries[key]; !ok { - hc.entries[key] = make(map[string]*TabletHealth) - hc.entries[key][tabletAlias] = hcc.tabletHealth + if ths, ok := hc.healthData[key]; !ok { + hc.healthData[key] = make(map[string]*TabletHealth) + hc.healthData[key][tabletAlias] = hcc.tabletHealth } else { if _, ok := ths[tabletAlias]; !ok { ths[tabletAlias] = hcc.tabletHealth @@ -662,7 +661,7 @@ func (hc *HealthCheckImpl) GetConnection(tabletAlias string) queryservice.QueryS } func (hc *HealthCheckImpl) findTabletHealthByAlias(alias string) *TabletHealth { - for _, ths := range hc.entries { + for _, ths := range hc.healthData { for _, th := range ths { if topoproto.TabletAliasString(th.Tablet.Alias) == alias { return th @@ -687,7 +686,7 @@ func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { tcsMap := make(map[string]*TabletsCacheStatus) hc.mu.Lock() defer hc.mu.Unlock() - for _, ths := range hc.entries { + for _, ths := range hc.healthData { for _, th := range ths { key := fmt.Sprintf("%v.%v.%v.%v", th.Tablet.Alias.Cell, th.Target.Keyspace, th.Target.Shard, th.Target.TabletType.String()) var tcs *TabletsCacheStatus @@ -708,12 +707,12 @@ func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { // Close stops the healthcheck. func (hc *HealthCheckImpl) Close() error { hc.mu.Lock() - for _, ths := range hc.entries { + for _, ths := range hc.healthData { for _, th := range ths { th.cancelFunc() } } - hc.entries = nil + hc.healthData = nil for _, tw := range hc.topoWatchers { tw.Stop() } @@ -754,11 +753,13 @@ func (hc *HealthCheckImpl) topologyWatcherChecksum() int64 { // The returned array is owned by the caller. // For TabletType_MASTER, this will only return at most one entry, // the most recent tablet of type master. +// This returns a copy of the data so that callers can access without +// synchronization func (hc *HealthCheckImpl) GetHealthyTabletStats(target *querypb.Target) []*TabletHealth { var result []*TabletHealth // we check all tablet types in all cells because of cellAliases key := hc.keyFromTarget(target) - ths, ok := hc.entries[key] + ths, ok := hc.healthData[key] if !ok { log.Warningf("Healthcheck has no tablet health for target: %v", key) return result @@ -769,11 +770,11 @@ func (hc *HealthCheckImpl) GetHealthyTabletStats(target *querypb.Target) []*Tabl } for _, th := range ths { if th.Tablet.Type == topodatapb.TabletType_MASTER { - result = append(result, th) + result = append(result, th.Copy()) return result } if th.isHealthy() { - result = append(result, th) + result = append(result, th.Copy()) } } return result @@ -785,10 +786,9 @@ func (hc *HealthCheckImpl) GetHealthyTabletStats(target *querypb.Target) []*Tabl // the most recent tablet of type master. func (hc *HealthCheckImpl) getTabletStats(target *querypb.Target) []*TabletHealth { var result []*TabletHealth - // we check all tablet types in all cells because of cellAliases - for _, ths := range hc.entries { + for _, ths := range hc.healthData { for _, th := range ths { - result = append(result, th) + result = append(result, th.Copy()) } } return result diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 6efbfd91a3a..bac424474ce 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -54,6 +54,28 @@ func (th *TabletHealth) String() string { th.Tablet, th.Target, th.Up, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) } +// Copy returns a copy of TabletHealth. Note that this is not really a deep copy +// because we point to the same underlying RealtimeStats. +// That is fine because the RealtimeStats object is never changed after creation. +func (th *TabletHealth) Copy() *TabletHealth { + th.mu.Lock() + defer th.mu.Unlock() + // we have to explicitly create a new object rather than relying on assignment to make a copy for us + // the following doesn't work for synchronized objects + // t := *th + // return &t + return &TabletHealth{ + Conn: th.Conn, + Tablet: th.Tablet, + Target: th.Target, + Up: th.Up, + Serving: th.Serving, + MasterTermStartTime: th.MasterTermStartTime, + Stats: th.Stats, + LastError: th.LastError, + } +} + // DeepEqual compares two TabletHealth. Since we include protos, we // need to use proto.Equal on these. func (th *TabletHealth) DeepEqual(other *TabletHealth) bool { diff --git a/go/vt/vtgate/buffer/buffer.go b/go/vt/vtgate/buffer/buffer.go index 08fcb05b52d..0327683d4a6 100644 --- a/go/vt/vtgate/buffer/buffer.go +++ b/go/vt/vtgate/buffer/buffer.go @@ -32,6 +32,8 @@ import ( "sync" "time" + querypb "vitess.io/vitess/go/vt/proto/query" + "golang.org/x/net/context" "vitess.io/vitess/go/sync2" @@ -213,6 +215,39 @@ func (b *Buffer) WaitForFailoverEnd(ctx context.Context, keyspace, shard string, return sb.waitForFailoverEnd(ctx, keyspace, shard, err) } +// WaitingForFailoverEnd tells us whether we are currently buffering +// which means someone might be waiting for the failover to end +func (b *Buffer) WaitingForFailoverEnd(ctx context.Context, keyspace, shard string) bool { + + sb := b.getOrCreateBuffer(keyspace, shard) + if sb == nil { + // buffer is stopped + return false + } + if sb.disabled() { + // no buffering is enabled so nothing to do + return false + } + + // only returns true if sb.state == stateBuffering + return sb.shouldBufferLocked(false) +} + +// EndFailover tells the buffer that the failover has ended +// so it can start retrying the buffered requests +func (b *Buffer) EndFailover(target *querypb.Target, th *discovery.TabletHealth) { + if target.TabletType != topodatapb.TabletType_MASTER { + panic(fmt.Sprintf("BUG: non MASTER target cannot end failover: %v", target)) + } + timestamp := th.MasterTermStartTime + sb := b.getOrCreateBuffer(target.Keyspace, target.Shard) + if sb == nil { + // Buffer is shut down. Ignore all calls. + return + } + sb.recordExternallyReparentedTimestamp(timestamp, th.Tablet.Alias) +} + // StatsUpdate keeps track of the "tablet_externally_reparented_timestamp" of // each master. This way we can detect the end of a failover. // It is part of the discovery.LegacyHealthCheckStatsListener interface. diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index 73b10ea15e6..a705af537d9 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -102,15 +102,6 @@ func (gw *TabletGateway) RegisterStats() { gw.hc.RegisterStats() } -// StatsUpdate forwards HealthCheck updates to TabletStatsCache and MasterBuffer. -// It is part of the discovery.HealthCheckStatsListener interface. -// TODO(deepthi): figure out how to update buffer -//func (gw *TabletGateway) StatsUpdate(ts *discovery.LegacyTabletStats) { -// if ts.Target.TabletType == topodatapb.TabletType_MASTER { -// gw.buffer.StatsUpdate(ts) -// } -//} - // WaitForTablets is part of the Gateway interface. func (gw *TabletGateway) WaitForTablets(ctx context.Context, tabletTypesToWait []topodatapb.TabletType) error { // Skip waiting for tablets if we are not told to do so. @@ -207,27 +198,27 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, gw.shuffleTablets(gw.localCell, tablets) var tabletLastUsed string - var conn queryservice.QueryService + var th *discovery.TabletHealth // skip tablets we tried before for _, t := range tablets { tabletLastUsed = topoproto.TabletAliasString(t.Tablet.Alias) if _, ok := invalidTablets[tabletLastUsed]; !ok { - conn = t.Conn + th = t break } else { tabletLastUsed = "" } } if tabletLastUsed == "" { + // do not override error from last attempt. if err == nil { - // do not override error from last attempt. err = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no available connection") } break } // execute - if conn == nil { + if th.Conn == nil { err = vterrors.Errorf(vtrpcpb.Code_UNAVAILABLE, "no connection for tablet %v", tabletLastUsed) invalidTablets[tabletLastUsed] = true continue @@ -235,8 +226,15 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, startTime := time.Now() var canRetry bool - canRetry, err = inner(ctx, target, conn) + canRetry, err = inner(ctx, target, th.Conn) gw.updateStats(target, startTime, err) + + // if a master query succeeded and the buffer is currently buffering, end the buffering + if err == nil && target.TabletType == topodatapb.TabletType_MASTER && gw.buffer.WaitingForFailoverEnd(ctx, target.Keyspace, target.Shard) { + // notify buffer that failover has ended + gw.buffer.EndFailover(target, th) + } + if canRetry { invalidTablets[tabletLastUsed] = true continue From 8a020f3ec87347e9f327efdf56c511761d8f656d Mon Sep 17 00:00:00 2001 From: deepthi Date: Wed, 29 Apr 2020 18:21:09 -0700 Subject: [PATCH 15/39] healthcheck: rename old tests to legacy Signed-off-by: deepthi --- ...lthcheck.go => fake_legacy_healthcheck.go} | 52 ++++++------- .../discovery/legacy_topology_watcher_test.go | 76 +++++++++---------- go/vt/discovery/tablet_health.go | 8 +- go/vt/vtexplain/vtexplain_vtgate.go | 4 +- go/vt/vtgate/discoverygateway_test.go | 10 +-- go/vt/vtgate/executor_framework_test.go | 4 +- go/vt/vtgate/executor_select_test.go | 22 +++--- go/vt/vtgate/executor_stream_test.go | 2 +- go/vt/vtgate/plan_executor_select_test.go | 22 +++--- go/vt/vtgate/scatter_conn_test.go | 12 +-- go/vt/vtgate/tx_conn_test.go | 2 +- go/vt/vtgate/vstream_manager_test.go | 20 ++--- go/vt/vtgate/vtgate_test.go | 4 +- 13 files changed, 119 insertions(+), 119 deletions(-) rename go/vt/discovery/{fake_healthcheck.go => fake_legacy_healthcheck.go} (72%) diff --git a/go/vt/discovery/fake_healthcheck.go b/go/vt/discovery/fake_legacy_healthcheck.go similarity index 72% rename from go/vt/discovery/fake_healthcheck.go rename to go/vt/discovery/fake_legacy_healthcheck.go index e19ce218c53..3abca835fd2 100644 --- a/go/vt/discovery/fake_healthcheck.go +++ b/go/vt/discovery/fake_legacy_healthcheck.go @@ -32,28 +32,28 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -// This file contains the definitions for a FakeHealthCheck class to +// This file contains the definitions for a FakeLegacyHealthCheck class to // simulate a LegacyHealthCheck module. Note it is not in a sub-package because // otherwise it couldn't be used in this package's tests because of // circular dependencies. -// NewFakeHealthCheck returns the fake healthcheck object. -func NewFakeHealthCheck() *FakeHealthCheck { - return &FakeHealthCheck{ - items: make(map[string]*fhcItem), +// NewFakeLegacyHealthCheck returns the fake healthcheck object. +func NewFakeLegacyHealthCheck() *FakeLegacyHealthCheck { + return &FakeLegacyHealthCheck{ + items: make(map[string]*flhcItem), } } -// FakeHealthCheck implements discovery.LegacyHealthCheck. -type FakeHealthCheck struct { +// FakeLegacyHealthCheck implements discovery.LegacyHealthCheck. +type FakeLegacyHealthCheck struct { listener LegacyHealthCheckStatsListener // mu protects the items map mu sync.RWMutex - items map[string]*fhcItem + items map[string]*flhcItem } -type fhcItem struct { +type flhcItem struct { ts *LegacyTabletStats conn queryservice.QueryService } @@ -63,22 +63,22 @@ type fhcItem struct { // // RegisterStats is not implemented. -func (fhc *FakeHealthCheck) RegisterStats() { +func (fhc *FakeLegacyHealthCheck) RegisterStats() { } // SetListener is not implemented. -func (fhc *FakeHealthCheck) SetListener(listener LegacyHealthCheckStatsListener, sendDownEvents bool) { +func (fhc *FakeLegacyHealthCheck) SetListener(listener LegacyHealthCheckStatsListener, sendDownEvents bool) { fhc.listener = listener } // WaitForInitialStatsUpdates is not implemented. -func (fhc *FakeHealthCheck) WaitForInitialStatsUpdates() { +func (fhc *FakeLegacyHealthCheck) WaitForInitialStatsUpdates() { } // AddTablet adds the tablet and calls the listener. -func (fhc *FakeHealthCheck) AddTablet(tablet *topodatapb.Tablet, name string) { +func (fhc *FakeLegacyHealthCheck) AddTablet(tablet *topodatapb.Tablet, name string) { key := TabletToMapKey(tablet) - item := &fhcItem{ + item := &flhcItem{ ts: &LegacyTabletStats{ Key: key, Tablet: tablet, @@ -104,7 +104,7 @@ func (fhc *FakeHealthCheck) AddTablet(tablet *topodatapb.Tablet, name string) { } // RemoveTablet removes the tablet. -func (fhc *FakeHealthCheck) RemoveTablet(tablet *topodatapb.Tablet) { +func (fhc *FakeLegacyHealthCheck) RemoveTablet(tablet *topodatapb.Tablet) { fhc.mu.Lock() defer fhc.mu.Unlock() key := TabletToMapKey(tablet) @@ -123,13 +123,13 @@ func (fhc *FakeHealthCheck) RemoveTablet(tablet *topodatapb.Tablet) { } // ReplaceTablet removes the old tablet and adds the new. -func (fhc *FakeHealthCheck) ReplaceTablet(old, new *topodatapb.Tablet, name string) { +func (fhc *FakeLegacyHealthCheck) ReplaceTablet(old, new *topodatapb.Tablet, name string) { fhc.RemoveTablet(old) fhc.AddTablet(new, name) } // GetConnection returns the TabletConn of the given tablet. -func (fhc *FakeHealthCheck) GetConnection(key string) queryservice.QueryService { +func (fhc *FakeLegacyHealthCheck) GetConnection(key string) queryservice.QueryService { fhc.mu.RLock() defer fhc.mu.RUnlock() if item := fhc.items[key]; item != nil { @@ -139,7 +139,7 @@ func (fhc *FakeHealthCheck) GetConnection(key string) queryservice.QueryService } // CacheStatus returns the status for each tablet -func (fhc *FakeHealthCheck) CacheStatus() LegacyTabletsCacheStatusList { +func (fhc *FakeLegacyHealthCheck) CacheStatus() LegacyTabletsCacheStatusList { fhc.mu.Lock() defer fhc.mu.Unlock() @@ -156,7 +156,7 @@ func (fhc *FakeHealthCheck) CacheStatus() LegacyTabletsCacheStatusList { } // Close is not implemented. -func (fhc *FakeHealthCheck) Close() error { +func (fhc *FakeLegacyHealthCheck) Close() error { return nil } @@ -165,18 +165,18 @@ func (fhc *FakeHealthCheck) Close() error { // // Reset cleans up the internal state. -func (fhc *FakeHealthCheck) Reset() { +func (fhc *FakeLegacyHealthCheck) Reset() { fhc.mu.Lock() defer fhc.mu.Unlock() - fhc.items = make(map[string]*fhcItem) + fhc.items = make(map[string]*flhcItem) } -// AddFakeTablet inserts a fake entry into FakeHealthCheck. +// AddFakeTablet inserts a fake entry into FakeLegacyHealthCheck. // The Tablet can be talked to using the provided connection. // The Listener is called, as if AddTablet had been called. // For flexibility the connection is created via a connFactory callback -func (fhc *FakeHealthCheck) AddFakeTablet(cell, host string, port int32, keyspace, shard string, tabletType topodatapb.TabletType, serving bool, reparentTS int64, err error, connFactory func(*topodatapb.Tablet) queryservice.QueryService) queryservice.QueryService { +func (fhc *FakeLegacyHealthCheck) AddFakeTablet(cell, host string, port int32, keyspace, shard string, tabletType topodatapb.TabletType, serving bool, reparentTS int64, err error, connFactory func(*topodatapb.Tablet) queryservice.QueryService) queryservice.QueryService { t := topo.NewTablet(0, cell, host) t.Keyspace = keyspace t.Shard = shard @@ -190,7 +190,7 @@ func (fhc *FakeHealthCheck) AddFakeTablet(cell, host string, port int32, keyspac defer fhc.mu.Unlock() item := fhc.items[key] if item == nil { - item = &fhcItem{ + item = &flhcItem{ ts: &LegacyTabletStats{ Key: key, Tablet: t, @@ -219,7 +219,7 @@ func (fhc *FakeHealthCheck) AddFakeTablet(cell, host string, port int32, keyspac // AddTestTablet adds a fake tablet for tests using the SandboxConn and returns // the fake connection -func (fhc *FakeHealthCheck) AddTestTablet(cell, host string, port int32, keyspace, shard string, tabletType topodatapb.TabletType, serving bool, reparentTS int64, err error) *sandboxconn.SandboxConn { +func (fhc *FakeLegacyHealthCheck) AddTestTablet(cell, host string, port int32, keyspace, shard string, tabletType topodatapb.TabletType, serving bool, reparentTS int64, err error) *sandboxconn.SandboxConn { conn := fhc.AddFakeTablet(cell, host, port, keyspace, shard, tabletType, serving, reparentTS, err, func(tablet *topodatapb.Tablet) queryservice.QueryService { return sandboxconn.NewSandboxConn(tablet) }) @@ -227,7 +227,7 @@ func (fhc *FakeHealthCheck) AddTestTablet(cell, host string, port int32, keyspac } // GetAllTablets returns all the tablets we have. -func (fhc *FakeHealthCheck) GetAllTablets() map[string]*topodatapb.Tablet { +func (fhc *FakeLegacyHealthCheck) GetAllTablets() map[string]*topodatapb.Tablet { res := make(map[string]*topodatapb.Tablet) fhc.mu.RLock() defer fhc.mu.RUnlock() diff --git a/go/vt/discovery/legacy_topology_watcher_test.go b/go/vt/discovery/legacy_topology_watcher_test.go index 8884e2fbdfb..06941bccfcb 100644 --- a/go/vt/discovery/legacy_topology_watcher_test.go +++ b/go/vt/discovery/legacy_topology_watcher_test.go @@ -29,7 +29,7 @@ import ( "vitess.io/vitess/go/vt/topo/memorytopo" ) -func checkOpCounts(t *testing.T, tw *LegacyTopologyWatcher, prevCounts, deltas map[string]int64) map[string]int64 { +func checkLegacyOpCounts(t *testing.T, tw *LegacyTopologyWatcher, prevCounts, deltas map[string]int64) map[string]int64 { t.Helper() newCounts := topologyWatcherOperations.Counts() for key, prevVal := range prevCounts { @@ -49,7 +49,7 @@ func checkOpCounts(t *testing.T, tw *LegacyTopologyWatcher, prevCounts, deltas m return newCounts } -func checkChecksum(t *testing.T, tw *LegacyTopologyWatcher, want uint32) { +func checkLegacyChecksum(t *testing.T, tw *LegacyTopologyWatcher, want uint32) { t.Helper() got := tw.TopoChecksum() if want != got { @@ -57,21 +57,21 @@ func checkChecksum(t *testing.T, tw *LegacyTopologyWatcher, want uint32) { } } -func TestCellTabletsWatcher(t *testing.T) { - checkWatcher(t, true, true) +func TestLegacyCellTabletsWatcher(t *testing.T) { + checkLegacyWatcher(t, true, true) } -func TestCellTabletsWatcherNoRefreshKnown(t *testing.T) { - checkWatcher(t, true, false) +func TestLegacyCellTabletsWatcherNoRefreshKnown(t *testing.T) { + checkLegacyWatcher(t, true, false) } -func TestShardReplicationWatcher(t *testing.T) { - checkWatcher(t, false, true) +func TestLegacyShardReplicationWatcher(t *testing.T) { + checkLegacyWatcher(t, false, true) } -func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { +func checkLegacyWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { ts := memorytopo.NewServer("aa") - fhc := NewFakeHealthCheck() + fhc := NewFakeLegacyHealthCheck() logger := logutil.NewMemoryLogger() topologyWatcherOperations.ZeroAll() counts := topologyWatcherOperations.Counts() @@ -88,8 +88,8 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { if err := tw.WaitForInitialTopology(); err != nil { t.Fatalf("initial WaitForInitialTopology failed") } - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1}) - checkChecksum(t, tw, 0) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1}) + checkLegacyChecksum(t, tw, 0) // Add a tablet to the topology. tablet := &topodatapb.Tablet{ @@ -108,8 +108,8 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { t.Fatalf("CreateTablet failed: %v", err) } tw.loadTablets() - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "AddTablet": 1}) - checkChecksum(t, tw, 1261153186) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "AddTablet": 1}) + checkLegacyChecksum(t, tw, 1261153186) // Check the tablet is returned by GetAllTablets(). allTablets := fhc.GetAllTablets() @@ -139,11 +139,11 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { // If RefreshKnownTablets is disabled, only the new tablet is read // from the topo if refreshKnownTablets { - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "AddTablet": 1}) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "AddTablet": 1}) } else { - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "AddTablet": 1}) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "AddTablet": 1}) } - checkChecksum(t, tw, 832404892) + checkLegacyChecksum(t, tw, 832404892) // Check the new tablet is returned by GetAllTablets(). allTablets = fhc.GetAllTablets() @@ -156,11 +156,11 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { // only the list is read from the topo and the checksum doesn't change tw.loadTablets() if refreshKnownTablets { - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2}) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2}) } else { - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1}) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1}) } - checkChecksum(t, tw, 832404892) + checkLegacyChecksum(t, tw, 832404892) // same tablet, different port, should update (previous // one should go away, new one be added) @@ -182,7 +182,7 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { key = TabletToMapKey(tablet) if refreshKnownTablets { - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 1}) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 1}) if _, ok := allTablets[key]; !ok || len(allTablets) != 2 || !proto.Equal(allTablets[key], tablet) { t.Errorf("fhc.GetAllTablets() = %+v; want %+v", allTablets, tablet) @@ -190,9 +190,9 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { if _, ok := allTablets[origKey]; ok { t.Errorf("fhc.GetAllTablets() = %+v; don't want %v", allTablets, origKey) } - checkChecksum(t, tw, 698548794) + checkLegacyChecksum(t, tw, 698548794) } else { - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1}) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1}) if _, ok := allTablets[origKey]; !ok || len(allTablets) != 2 || !proto.Equal(allTablets[origKey], origTablet) { t.Errorf("fhc.GetAllTablets() = %+v; want %+v", allTablets, origTablet) @@ -200,7 +200,7 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { if _, ok := allTablets[key]; ok { t.Errorf("fhc.GetAllTablets() = %+v; don't want %v", allTablets, key) } - checkChecksum(t, tw, 832404892) + checkLegacyChecksum(t, tw, 832404892) } // Remove the second tablet and re-add with a new uid. This should @@ -224,11 +224,11 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { allTablets = fhc.GetAllTablets() if refreshKnownTablets { - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 1}) - checkChecksum(t, tw, 4097170367) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 1}) + checkLegacyChecksum(t, tw, 4097170367) } else { - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "ReplaceTablet": 1}) - checkChecksum(t, tw, 3960185881) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "ReplaceTablet": 1}) + checkLegacyChecksum(t, tw, 3960185881) } key = TabletToMapKey(tablet2) if _, ok := allTablets[key]; !ok || len(allTablets) != 2 || !proto.Equal(allTablets[key], tablet2) { @@ -258,7 +258,7 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { t.Fatalf("UpdateTabletFields failed: %v", err) } tw.loadTablets() - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 2}) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 2}) allTablets = fhc.GetAllTablets() key2 := TabletToMapKey(tablet2) if _, ok := allTablets[key2]; !ok { @@ -282,7 +282,7 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { t.Fatalf("UpdateTabletFields failed: %v", err) } tw.loadTablets() - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 2}) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 2}) } // Remove the tablet and check that it is detected as being gone. @@ -294,11 +294,11 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { } tw.loadTablets() if refreshKnownTablets { - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "RemoveTablet": 1}) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "RemoveTablet": 1}) } else { - counts = checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "RemoveTablet": 1}) + counts = checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "RemoveTablet": 1}) } - checkChecksum(t, tw, 1725545897) + checkLegacyChecksum(t, tw, 1725545897) allTablets = fhc.GetAllTablets() key = TabletToMapKey(tablet) @@ -318,8 +318,8 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { t.Fatalf("FixShardReplication failed: %v", err) } tw.loadTablets() - checkOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 0, "RemoveTablet": 1}) - checkChecksum(t, tw, 0) + checkLegacyOpCounts(t, tw, counts, map[string]int64{"ListTablets": 1, "GetTablet": 0, "RemoveTablet": 1}) + checkLegacyChecksum(t, tw, 0) allTablets = fhc.GetAllTablets() key = TabletToMapKey(tablet) @@ -334,7 +334,7 @@ func checkWatcher(t *testing.T, cellTablets, refreshKnownTablets bool) { tw.Stop() } -func TestFilterByShard(t *testing.T) { +func TestLegacyFilterByShard(t *testing.T) { testcases := []struct { filters []string keyspace string @@ -431,8 +431,8 @@ var ( testHostName = "testHostName" ) -func TestFilterByKeyspace(t *testing.T) { - hc := NewFakeHealthCheck() +func TestLegacyFilterByKeyspace(t *testing.T) { + hc := NewFakeLegacyHealthCheck() tr := NewLegacyFilterByKeyspace(hc, testKeyspacesToWatch) ts := memorytopo.NewServer(testCell) tw := NewLegacyCellTabletsWatcher(context.Background(), ts, tr, testCell, 10*time.Minute, true, 5) diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index bac424474ce..27ac8911274 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -18,14 +18,14 @@ import ( // TabletHealth maintains the health status of a tablet. A map of this // structure is maintained in HealthCheckImpl. type TabletHealth struct { - mu sync.Mutex // cancelFunc must be called before discarding TabletHealth. // This will ensure that the associated checkConn goroutine will terminate. cancelFunc context.CancelFunc - // Conn is the connection associated with the tablet. - Conn queryservice.QueryService // Tablet is the tablet object that was sent to HealthCheck.AddTablet. Tablet *topodata.Tablet + mu sync.Mutex + // Conn is the connection associated with the tablet. + Conn queryservice.QueryService // Target is the current target as returned by the streaming // StreamHealth RPC. Target *query.Target @@ -135,9 +135,9 @@ func (th *TabletHealth) getTabletDebugURL() string { func (th *TabletHealth) deleteConnLocked() { th.mu.Lock() - defer th.mu.Unlock() th.Up = false th.Conn = nil + th.mu.Unlock() th.cancelFunc() } diff --git a/go/vt/vtexplain/vtexplain_vtgate.go b/go/vt/vtexplain/vtexplain_vtgate.go index bee40e18e2d..a59c8b2d0cf 100644 --- a/go/vt/vtexplain/vtexplain_vtgate.go +++ b/go/vt/vtexplain/vtexplain_vtgate.go @@ -42,7 +42,7 @@ import ( var ( explainTopo *ExplainTopo vtgateExecutor *vtgate.Executor - healthCheck *discovery.FakeHealthCheck + healthCheck *discovery.FakeLegacyHealthCheck vtgateSession = &vtgatepb.Session{ TargetString: "", @@ -52,7 +52,7 @@ var ( func initVtgateExecutor(vSchemaStr string, opts *Options) error { explainTopo = &ExplainTopo{NumShards: opts.NumShards} - healthCheck = discovery.NewFakeHealthCheck() + healthCheck = discovery.NewFakeLegacyHealthCheck() resolver := newFakeResolver(opts, healthCheck, explainTopo, vtexplainCell) diff --git a/go/vt/vtgate/discoverygateway_test.go b/go/vt/vtgate/discoverygateway_test.go index 96ada8bfe54..b02ec1fa471 100644 --- a/go/vt/vtgate/discoverygateway_test.go +++ b/go/vt/vtgate/discoverygateway_test.go @@ -106,7 +106,7 @@ func TestDiscoveryGatewayBeginExecuteBatch(t *testing.T) { func TestDiscoveryGatewayGetTablets(t *testing.T) { keyspace := "ks" shard := "0" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() dg := NewDiscoveryGateway(context.Background(), hc, nil, "local", 2) // replica should only use local ones @@ -206,7 +206,7 @@ func TestShuffleTablets(t *testing.T) { func TestDiscoveryGatewayGetTabletsInRegion(t *testing.T) { keyspace := "ks" shard := "0" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() ts := memorytopo.NewServer("local-west", "local-east", "local", "remote") srvTopo := srvtopotest.NewPassthroughSrvTopoServer() srvTopo.TopoServer = ts @@ -236,7 +236,7 @@ func TestDiscoveryGatewayGetTabletsInRegion(t *testing.T) { func TestDiscoveryGatewayGetTabletsWithRegion(t *testing.T) { keyspace := "ks" shard := "0" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() ts := memorytopo.NewServer("local-west", "local-east", "local", "remote") srvTopo := srvtopotest.NewPassthroughSrvTopoServer() srvTopo.TopoServer = ts @@ -273,7 +273,7 @@ func testDiscoveryGatewayGeneric(t *testing.T, f func(dg *DiscoveryGateway, targ Shard: shard, TabletType: tabletType, } - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() dg := NewDiscoveryGateway(context.Background(), hc, nil, "cell", 2) // no tablet @@ -356,7 +356,7 @@ func testDiscoveryGatewayTransact(t *testing.T, f func(dg *DiscoveryGateway, tar Shard: shard, TabletType: tabletType, } - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() dg := NewDiscoveryGateway(context.Background(), hc, nil, "cell", 2) // retry error - no retry diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index a4e3e833c96..df7cba44c2d 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -343,7 +343,7 @@ const ( func createExecutorEnvUsing(t executorType) (executor *Executor, sbc1, sbc2, sbclookup *sandboxconn.SandboxConn) { cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema serv := newSandboxForCells([]string{cell}) @@ -387,7 +387,7 @@ func createExecutorEnv() (executor *Executor, sbc1, sbc2, sbclookup *sandboxconn func createCustomExecutor(vschema string) (executor *Executor, sbc1, sbc2, sbclookup *sandboxconn.SandboxConn) { cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = vschema serv := newSandboxForCells([]string{cell}) diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index 163d661e78d..398b2f7180a 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -894,7 +894,7 @@ func TestStreamSelectIN(t *testing.T) { func TestSelectScatter(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -927,7 +927,7 @@ func TestSelectScatter(t *testing.T) { func TestSelectScatterPartial(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -987,7 +987,7 @@ func TestSelectScatterPartial(t *testing.T) { func TestStreamSelectScatter(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1024,7 +1024,7 @@ func TestStreamSelectScatter(t *testing.T) { func TestSelectScatterOrderBy(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1094,7 +1094,7 @@ func TestSelectScatterOrderBy(t *testing.T) { func TestSelectScatterOrderByVarChar(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1164,7 +1164,7 @@ func TestSelectScatterOrderByVarChar(t *testing.T) { func TestStreamSelectScatterOrderBy(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1225,7 +1225,7 @@ func TestStreamSelectScatterOrderBy(t *testing.T) { func TestStreamSelectScatterOrderByVarChar(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1288,7 +1288,7 @@ func TestStreamSelectScatterOrderByVarChar(t *testing.T) { func TestSelectScatterAggregate(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1351,7 +1351,7 @@ func TestSelectScatterAggregate(t *testing.T) { func TestStreamSelectScatterAggregate(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1414,7 +1414,7 @@ func TestStreamSelectScatterAggregate(t *testing.T) { func TestSelectScatterLimit(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1486,7 +1486,7 @@ func TestSelectScatterLimit(t *testing.T) { func TestStreamSelectScatterLimit(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema diff --git a/go/vt/vtgate/executor_stream_test.go b/go/vt/vtgate/executor_stream_test.go index ba708f4b724..45cc8068c81 100644 --- a/go/vt/vtgate/executor_stream_test.go +++ b/go/vt/vtgate/executor_stream_test.go @@ -47,7 +47,7 @@ func TestStreamSQLUnsharded(t *testing.T) { func TestStreamSQLSharded(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema diff --git a/go/vt/vtgate/plan_executor_select_test.go b/go/vt/vtgate/plan_executor_select_test.go index 4a8caea274e..5c625698ec6 100644 --- a/go/vt/vtgate/plan_executor_select_test.go +++ b/go/vt/vtgate/plan_executor_select_test.go @@ -871,7 +871,7 @@ func TestPlanStreamSelectIN(t *testing.T) { func TestPlanSelectScatter(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -904,7 +904,7 @@ func TestPlanSelectScatter(t *testing.T) { func TestPlanSelectScatterPartial(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -964,7 +964,7 @@ func TestPlanSelectScatterPartial(t *testing.T) { func TestPlanStreamSelectScatter(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1001,7 +1001,7 @@ func TestPlanStreamSelectScatter(t *testing.T) { func TestPlanSelectScatterOrderBy(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1071,7 +1071,7 @@ func TestPlanSelectScatterOrderBy(t *testing.T) { func TestPlanSelectScatterOrderByVarChar(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1141,7 +1141,7 @@ func TestPlanSelectScatterOrderByVarChar(t *testing.T) { func TestPlanStreamSelectScatterOrderBy(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1202,7 +1202,7 @@ func TestPlanStreamSelectScatterOrderBy(t *testing.T) { func TestPlanStreamSelectScatterOrderByVarChar(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1265,7 +1265,7 @@ func TestPlanStreamSelectScatterOrderByVarChar(t *testing.T) { func TestPlanSelectScatterAggregate(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1328,7 +1328,7 @@ func TestPlanSelectScatterAggregate(t *testing.T) { func TestPlanStreamSelectScatterAggregate(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1391,7 +1391,7 @@ func TestPlanStreamSelectScatterAggregate(t *testing.T) { func TestPlanSelectScatterLimit(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema @@ -1463,7 +1463,7 @@ func TestPlanSelectScatterLimit(t *testing.T) { func TestPlanStreamSelectScatterLimit(t *testing.T) { // Special setup: Don't use createExecutorEnv. cell := "aa" - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") s.VSchema = executorVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema diff --git a/go/vt/vtgate/scatter_conn_test.go b/go/vt/vtgate/scatter_conn_test.go index 3e905ee999a..b14deea5890 100644 --- a/go/vt/vtgate/scatter_conn_test.go +++ b/go/vt/vtgate/scatter_conn_test.go @@ -119,7 +119,7 @@ func verifyScatterConnError(t *testing.T, err error, wantErr string, wantCode vt } func testScatterConnGeneric(t *testing.T, name string, f func(sc *ScatterConn, shards []string) (*sqltypes.Result, error)) { - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() // no shard s := createSandbox(name) @@ -230,7 +230,7 @@ func TestMaxMemoryRows(t *testing.T) { defer func() { *maxMemoryRows = save }() createSandbox("TestMaxMemoryRows") - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() sc := newTestScatterConn(hc, new(sandboxTopo), "aa") sbc0 := hc.AddTestTablet("aa", "0", 1, "TestMaxMemoryRows", "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1", 1, "TestMaxMemoryRows", "1", topodatapb.TabletType_REPLICA, true, 1, nil) @@ -277,7 +277,7 @@ func TestMaxMemoryRows(t *testing.T) { func TestMultiExecs(t *testing.T) { createSandbox("TestMultiExecs") - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() sc := newTestScatterConn(hc, new(sandboxTopo), "aa") sbc0 := hc.AddTestTablet("aa", "0", 1, "TestMultiExecs", "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1", 1, "TestMultiExecs", "1", topodatapb.TabletType_REPLICA, true, 1, nil) @@ -369,7 +369,7 @@ func TestMultiExecs(t *testing.T) { func TestScatterConnStreamExecuteSendError(t *testing.T) { createSandbox("TestScatterConnStreamExecuteSendError") - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() sc := newTestScatterConn(hc, new(sandboxTopo), "aa") hc.AddTestTablet("aa", "0", 1, "TestScatterConnStreamExecuteSendError", "0", topodatapb.TabletType_REPLICA, true, 1, nil) res := srvtopo.NewResolver(&sandboxTopo{}, sc.gateway, "aa") @@ -389,7 +389,7 @@ func TestScatterConnStreamExecuteSendError(t *testing.T) { func TestScatterConnQueryNotInTransaction(t *testing.T) { s := createSandbox("TestScatterConnQueryNotInTransaction") - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() // case 1: read query (not in transaction) followed by write query, not in the same shard. hc.Reset() @@ -543,7 +543,7 @@ func TestScatterConnQueryNotInTransaction(t *testing.T) { func TestScatterConnSingleDB(t *testing.T) { createSandbox("TestScatterConnSingleDB") - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() hc.Reset() sc := newTestScatterConn(hc, new(sandboxTopo), "aa") diff --git a/go/vt/vtgate/tx_conn_test.go b/go/vt/vtgate/tx_conn_test.go index b7a945a2642..e45d207d946 100644 --- a/go/vt/vtgate/tx_conn_test.go +++ b/go/vt/vtgate/tx_conn_test.go @@ -914,7 +914,7 @@ func TestTxConnMultiGoTargets(t *testing.T) { func newTestTxConnEnv(t *testing.T, name string) (sc *ScatterConn, sbc0, sbc1 *sandboxconn.SandboxConn, rss0, rss1, rss01 []*srvtopo.ResolvedShard) { createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() sc = newTestScatterConn(hc, new(sandboxTopo), "aa") sbc0 = hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 = hc.AddTestTablet("aa", "1", 1, name, "1", topodatapb.TabletType_MASTER, true, 1, nil) diff --git a/go/vt/vtgate/vstream_manager_test.go b/go/vt/vtgate/vstream_manager_test.go index 6779a9b889c..f13e4f71309 100644 --- a/go/vt/vtgate/vstream_manager_test.go +++ b/go/vt/vtgate/vstream_manager_test.go @@ -40,7 +40,7 @@ func TestVStreamEvents(t *testing.T) { name := "TestVStream" _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() vsm := newTestVStreamManager(hc, new(sandboxTopo), "aa") sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) @@ -114,7 +114,7 @@ func TestVStreamChunks(t *testing.T) { name := "TestVStream" _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() vsm := newTestVStreamManager(hc, new(sandboxTopo), "aa") sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) @@ -184,7 +184,7 @@ func TestVStreamMulti(t *testing.T) { name := "TestVStream" _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() vsm := newTestVStreamManager(hc, new(sandboxTopo), "aa") sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) @@ -243,7 +243,7 @@ func TestVStreamRetry(t *testing.T) { name := "TestVStream" _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() vsm := newTestVStreamManager(hc, new(sandboxTopo), "aa") sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) @@ -282,7 +282,7 @@ func TestVStreamHeartbeat(t *testing.T) { name := "TestVStream" _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() vsm := newTestVStreamManager(hc, new(sandboxTopo), "aa") sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) @@ -330,7 +330,7 @@ func TestVStreamJournalOneToMany(t *testing.T) { name := "TestVStream" _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() vsm := newTestVStreamManager(hc, new(sandboxTopo), "aa") sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "-10", topodatapb.TabletType_MASTER, true, 1, nil) @@ -435,7 +435,7 @@ func TestVStreamJournalManyToOne(t *testing.T) { // Variable names are maintained like in OneToMany, but order is different. name := "TestVStream" _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() vsm := newTestVStreamManager(hc, new(sandboxTopo), "aa") sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "-10", topodatapb.TabletType_MASTER, true, 1, nil) @@ -544,7 +544,7 @@ func TestVStreamJournalNoMatch(t *testing.T) { name := "TestVStream" _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() vsm := newTestVStreamManager(hc, new(sandboxTopo), "aa") sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) @@ -670,7 +670,7 @@ func TestVStreamJournalPartialMatch(t *testing.T) { // Variable names are maintained like in OneToMany, but order is different.1 name := "TestVStream" _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() vsm := newTestVStreamManager(hc, new(sandboxTopo), "aa") _ = hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "-10", topodatapb.TabletType_MASTER, true, 1, nil) sbc2 := hc.AddTestTablet("aa", "1.1.1.1", 1003, name, "10-20", topodatapb.TabletType_MASTER, true, 1, nil) @@ -748,7 +748,7 @@ func TestVStreamJournalPartialMatch(t *testing.T) { func TestResolveVStreamParams(t *testing.T) { name := "TestVStream" _ = createSandbox(name) - hc := discovery.NewFakeHealthCheck() + hc := discovery.NewFakeLegacyHealthCheck() vsm := newTestVStreamManager(hc, new(sandboxTopo), "aa") testcases := []struct { input *binlogdatapb.VGtid diff --git a/go/vt/vtgate/vtgate_test.go b/go/vt/vtgate/vtgate_test.go index 16f11b025ab..12befb2aabf 100644 --- a/go/vt/vtgate/vtgate_test.go +++ b/go/vt/vtgate/vtgate_test.go @@ -37,7 +37,7 @@ import ( // This file uses the sandbox_test framework. -var hcVTGateTest *discovery.FakeHealthCheck +var hcVTGateTest *discovery.FakeLegacyHealthCheck var executeOptions = &querypb.ExecuteOptions{ IncludedFields: querypb.ExecuteOptions_TYPE_ONLY, @@ -69,7 +69,7 @@ func init() { } } ` - hcVTGateTest = discovery.NewFakeHealthCheck() + hcVTGateTest = discovery.NewFakeLegacyHealthCheck() *transactionMode = "MULTI" // The topo.Server is used to start watching the cells described // in '-cells_to_watch' command line parameter, which is From 543dd23228194a465b4352c524419e047d012c61 Mon Sep 17 00:00:00 2001 From: deepthi Date: Fri, 1 May 2020 20:59:00 -0700 Subject: [PATCH 16/39] healthcheck: move loadTablets from TopologyWatcher to HealthCheckImpl, implement healthy tablet sorting by replica lag, notify buffer when master changes. Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 371 +++++++++++++++++------ go/vt/discovery/healthcheck_test.go | 4 +- go/vt/discovery/legacy_replicationlag.go | 18 +- go/vt/discovery/replicationlag.go | 111 ++++++- go/vt/discovery/tablet_health.go | 9 +- go/vt/discovery/tablets_cache_status.go | 3 - go/vt/discovery/topology_watcher.go | 197 +----------- go/vt/servenv/status.go | 2 + go/vt/vtgate/buffer/buffer.go | 35 +-- go/vt/vtgate/tabletgateway.go | 16 +- 10 files changed, 428 insertions(+), 338 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index f148ad9e217..f417d4ee844 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -56,7 +56,6 @@ import ( "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools" - "vitess.io/vitess/go/vt/vttablet/queryservice" "vitess.io/vitess/go/vt/vttablet/tabletconn" querypb "vitess.io/vitess/go/vt/proto/query" @@ -153,20 +152,6 @@ func init() { flag.Var(&KeyspacesToWatch, "keyspaces_to_watch", "Specifies which keyspaces this vtgate should have access to while routing queries or accessing the vschema") } -// TabletRecorder is the part of the HealthCheck interface that can -// add or remove tablets. We define it as a sub-interface here so -// that it can be implemented by other users of TopologyWatcher -type TabletRecorder interface { - // AddTablet adds the tablet. - AddTablet(tablet *topodatapb.Tablet) - - // RemoveTablet removes the tablet. - RemoveTablet(tablet *topodatapb.Tablet) - - // ReplaceTablet does an AddTablet and RemoveTablet in one call, effectively replacing the old tablet with the new. - ReplaceTablet(old, new *topodatapb.Tablet) -} - // HealthCheck defines the interface of health checking module. // The goal of this object is to maintain a StreamHealth RPC // to a lot of tablets. Tablets are added / removed by calling the @@ -184,6 +169,12 @@ type HealthCheck interface { GetHealthyTabletStats(target *querypb.Target) []*TabletHealth // WaitForAllServingTablets allows vtgate to wait for all tablets to be serving before accepting requests WaitForAllServingTablets(ctx context.Context, targets []*querypb.Target) error + // AddTablet adds the tablet. + AddTablet(tablet *topodatapb.Tablet) + // RemoveTablet removes the tablet. + RemoveTablet(tablet *topodatapb.Tablet) + // ReplaceTablet does an AddTablet and RemoveTablet in one call, effectively replacing the old tablet with the new. + ReplaceTablet(old, new *topodatapb.Tablet) } // HealthCheckImpl performs health checking and stores the results. @@ -202,16 +193,31 @@ type HealthCheckImpl struct { cell string // mu protects all the following fields. mu sync.Mutex - + // authoritative map of tabletHealth by alias + healthByAlias map[string]*TabletHealth // a map keyed by keyspace.shard.tabletType // contains a map of TabletHealth keyed by tablet alias for each tablet relevant to the keyspace.shard.tabletType + // has to be kept in sync with healthByAlias healthData map[string]map[string]*TabletHealth + // another map keyed by keyspace.shard.tabletType, this one containing a sorted list of TabletHealth + // TODO(deepthi): replace with SimpleTabletHealth + healthy map[string][]*TabletHealth // connsWG keeps track of all launched Go routines that monitor tablet connections. connsWG sync.WaitGroup // topology watchers that inform healthcheck of tablets being added and deleted topoWatchers []*TopologyWatcher + // used to inform vtgate buffer when new master is detected + // TODO: replace this with synchronizing over a condition variable + masterCallback func(health *TabletHealth) + // cellAliases is a cache of cell aliases + cellAliases map[string]string } +//type SimpleTabletHealth struct { +// TabletAlias string +// Conn queryservice.QueryService +//} + // HealthCheckConn is a structure that lives within the scope of // the checkConn goroutine to maintain its internal state. Therefore, // it does not require synchronization. Changes that are relevant to @@ -238,7 +244,7 @@ type healthCheckConn struct { // The topology server that this healthcheck object can use to retrieve cell or tablet information // localCell. // The localCell for this healthcheck -func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string) HealthCheck { +func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string, callback func(health *TabletHealth)) HealthCheck { log.Infof("loading tablets for cells: %v", *CellsToWatch) hc := &HealthCheckImpl{ @@ -246,6 +252,10 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur cell: localCell, retryDelay: retryDelay, healthCheckTimeout: healthCheckTimeout, + masterCallback: callback, + healthByAlias: make(map[string]*TabletHealth), + healthData: make(map[string]map[string]*TabletHealth), + healthy: make(map[string][]*TabletHealth), } var topoWatchers []*TopologyWatcher var filter TabletFilter @@ -254,6 +264,7 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur cells = append(cells, localCell) } for _, c := range cells { + log.Infof("Setting up healthcheck for cell: %v", c) if c == "" { continue } @@ -270,7 +281,7 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur } else if len(KeyspacesToWatch) > 0 { filter = NewFilterByKeyspace(c, KeyspacesToWatch) } - topoWatchers = append(topoWatchers, NewCellTabletsWatcher(ctx, topoServer, hc, filter, c, *RefreshInterval, *RefreshKnownTablets, *TopoReadConcurrency)) + topoWatchers = append(topoWatchers, NewCellTabletsWatcher(ctx, topoServer, filter, c, *RefreshInterval, *RefreshKnownTablets, *TopoReadConcurrency)) } hc.topoWatchers = topoWatchers @@ -280,12 +291,171 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur // start the topo watches here for _, tw := range hc.topoWatchers { - go tw.watchTopo() + go hc.watchTopo(tw) } return hc } +func (hc *HealthCheckImpl) watchTopo(tw *TopologyWatcher) { + tw.wg.Add(1) + defer tw.wg.Done() + ticker := time.NewTicker(tw.refreshInterval) + defer ticker.Stop() + for { + hc.loadTablets(tw) + select { + case <-tw.ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (hc *HealthCheckImpl) loadTablets(tw *TopologyWatcher) { + var wg sync.WaitGroup + newTablets := make(map[string]*tabletInfo) + + // first get the list of relevant tabletAliases + tabletAliases, err := tw.getTablets(tw) + topologyWatcherOperations.Add(topologyWatcherOpListTablets, 1) + if err != nil { + topologyWatcherErrors.Add(topologyWatcherOpListTablets, 1) + select { + case <-tw.ctx.Done(): + return + default: + } + log.Errorf("cannot get tablets for cell: %v: %v", tw.cell, err) + return + } + + // Accumulate a list of all known alias strings to use later + // when sorting + tabletAliasStrs := make([]string, 0, len(tabletAliases)) + + tw.mu.Lock() + for _, tAlias := range tabletAliases { + aliasStr := topoproto.TabletAliasString(tAlias) + tabletAliasStrs = append(tabletAliasStrs, aliasStr) + + if !tw.refreshKnownTablets { + // we already have a tabletInfo for this and the flag tells us to not refresh + if val, ok := tw.tablets[aliasStr]; ok { + newTablets[aliasStr] = val + continue + } + } + + wg.Add(1) + go func(alias *topodatapb.TabletAlias) { + defer wg.Done() + tw.sem <- 1 // Wait for active queue to drain. + tablet, err := tw.topoServer.GetTablet(tw.ctx, alias) + topologyWatcherOperations.Add(topologyWatcherOpGetTablet, 1) + <-tw.sem // Done; enable next request to run + if err != nil { + topologyWatcherErrors.Add(topologyWatcherOpGetTablet, 1) + select { + case <-tw.ctx.Done(): + return + default: + } + log.Errorf("cannot get tablet for alias %v: %v", alias, err) + return + } + if !(hc.isTabletInCell(tablet.Tablet) && (tw.tabletFilter == nil || tw.tabletFilter.IsIncluded(tablet.Tablet))) { + log.Errorf("loadTablets skipping tablet: %#v", tablet.Tablet) + return + } + tw.mu.Lock() + aliasStr := topoproto.TabletAliasString(alias) + newTablets[aliasStr] = &tabletInfo{ + alias: aliasStr, + tablet: tablet.Tablet, + } + tw.mu.Unlock() + }(tAlias) + } + + tw.mu.Unlock() + wg.Wait() + tw.mu.Lock() + + for alias, newVal := range newTablets { + // trust the alias from topo and add it if it doesn't exist + if val, ok := tw.tablets[alias]; !ok { + hc.AddTablet(newVal.tablet) + topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1) + } else { + // check if the host and port have changed. If yes, replace tablet + oldKey := TabletToMapKey(val.tablet) + newKey := TabletToMapKey(newVal.tablet) + if oldKey != newKey { + // This is the case where the same tablet alias is now reporting + // a different address key. + hc.ReplaceTablet(val.tablet, newVal.tablet) + topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) + } + } + } + + for _, val := range tw.tablets { + if _, ok := newTablets[val.alias]; !ok { + hc.RemoveTablet(val.tablet) + topologyWatcherOperations.Add(topologyWatcherOpRemoveTablet, 1) + } + } + tw.tablets = newTablets + if !tw.firstLoadDone { + tw.firstLoadDone = true + close(tw.firstLoadChan) + } + + // iterate through the tablets in a stable order and compute a + // checksum of the tablet map + sort.Strings(tabletAliasStrs) + var buf bytes.Buffer + for _, alias := range tabletAliasStrs { + _, ok := tw.tablets[alias] + if ok { + buf.WriteString(alias) + } + } + tw.topoChecksum = crc32.ChecksumIEEE(buf.Bytes()) + tw.lastRefresh = time.Now() + + tw.mu.Unlock() + +} + +func (hc *HealthCheckImpl) getAliasByCell(cell string) string { + hc.mu.Lock() + defer hc.mu.Unlock() + + if alias, ok := hc.cellAliases[cell]; ok { + return alias + } + + alias := topo.GetAliasByCell(context.Background(), hc.ts, cell) + hc.cellAliases[cell] = alias + + return alias +} + +func (hc *HealthCheckImpl) isTabletInCell(tablet *topodatapb.Tablet) bool { + if tablet.Type == topodatapb.TabletType_MASTER { + return true + } + if tablet.Alias.Cell == hc.cell { + return true + } + if hc.getAliasByCell(tablet.Alias.Cell) == hc.getAliasByCell(hc.cell) { + return true + } + return false +} + // RegisterStats registers the connection counts stats func (hc *HealthCheckImpl) RegisterStats() { stats.NewGaugeDurationFunc( @@ -315,7 +485,7 @@ func (hc *HealthCheckImpl) RegisterStats() { // ServeHTTP is part of the http.Handler interface. It renders the current state of the discovery gateway tablet cache into json. func (hc *HealthCheckImpl) ServeHTTP(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json; charset=utf-8") - status := hc.cacheStatusMap() + status := hc.CacheStatus() b, err := json.MarshalIndent(status, "", " ") if err != nil { w.Write([]byte(err.Error())) @@ -334,7 +504,7 @@ func (hc *HealthCheckImpl) servingConnStats() map[string]int64 { defer hc.mu.Unlock() for key, ths := range hc.healthData { for _, th := range ths { - if !th.Up || !th.Serving || th.LastError != nil { + if !th.Serving || th.LastError != nil { continue } res[key]++ @@ -358,7 +528,7 @@ func (hc *HealthCheckImpl) stateChecksum() int64 { ) sort.Sort(st.TabletsStats) for _, ts := range st.TabletsStats { - fmt.Fprintf(&buf, "%v%v%v\n", ts.Up, ts.Serving, ts.MasterTermStartTime) + fmt.Fprintf(&buf, "%v%v\n", ts.Serving, ts.MasterTermStartTime) } } @@ -371,7 +541,6 @@ func (hc *HealthCheckImpl) stateChecksum() int64 { func (hc *HealthCheckImpl) finalizeConn(hcc *healthCheckConn) { hcc.tabletHealth.mu.Lock() defer hcc.tabletHealth.mu.Unlock() - hcc.tabletHealth.Up = false hcc.setServingState(false, "finalizeConn closing connection") // Note: checkConn() exits only when hcc.ctx.Done() is closed. Thus it's // safe to simply get Err() value here and assign to LastError. @@ -536,15 +705,30 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St } if shr.TabletAlias != nil && !proto.Equal(shr.TabletAlias, hcc.tabletHealth.Tablet.Alias) { + // TabletAlias change means that the host:port has been taken over by another tablet + // We could cancel / exit the healthcheck for this tablet right away + // However, we defer it until the next topo refresh informs us of the change because that is + // the only way to discover the new host/port return fmt.Errorf("health stats mismatch, tablet %+v alias does not match response alias %v", hcc.tabletHealth.Tablet, shr.TabletAlias) } hcc.tabletHealth.mu.Lock() currentTablet := hcc.tabletHealth.Tablet + // check whether this is a trivial update so as to update healthy map + trivialNonMasterUpdate := hcc.tabletHealth.LastError == nil && hcc.tabletHealth.Serving && shr.RealtimeStats.HealthError == "" && shr.Serving && + currentTablet.Type != topodatapb.TabletType_MASTER && currentTablet.Type == shr.Target.TabletType + isMasterUpdate := currentTablet.Type == topodatapb.TabletType_MASTER && shr.Target.TabletType == topodatapb.TabletType_MASTER hcc.tabletHealth.mu.Unlock() + + // hc.healthByAlias is authoritative, it should be updated + hc.mu.Lock() + tabletAlias := topoproto.TabletAliasString(hcc.tabletHealth.Tablet.Alias) + // this will only change the first time, but it's easiest to set it always rather than check and set + hc.healthByAlias[tabletAlias] = hcc.tabletHealth + hc.mu.Unlock() // In this case where a new tablet is initialized or a tablet type changes, we want to // initialize the counter so the rate can be calculated correctly. - if currentTablet.Type != shr.Target.TabletType { + if currentTablet.Type != shr.Target.TabletType || currentTablet.Keyspace != shr.Target.Keyspace || currentTablet.Shard != shr.Target.Shard { hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) // hc still has this TabletHealth in the wrong target (because tabletType changed) oldTargetKey := hc.keyFromTablet(currentTablet) @@ -556,8 +740,7 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St hc.mu.Unlock() } - // Update our record, and notify downstream for tabletType and - // realtimeStats change. + // Update our record hcc.lastResponseTimestamp = time.Now() hcc.tabletHealth.mu.Lock() defer hcc.tabletHealth.mu.Unlock() @@ -570,6 +753,41 @@ func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.St reason = "healthCheck update error: " + healthErr.Error() } hcc.setServingState(serving, reason) + + targetKey := hc.keyFromTarget(shr.Target) + if !trivialNonMasterUpdate { + all := hc.healthData[targetKey] + allArray := make([]*TabletHealth, 0, len(all)) + for _, s := range all { + allArray = append(allArray, s) + } + hc.healthy[targetKey] = FilterStatsByReplicationLag(allArray) + } + if isMasterUpdate { + if len(hc.healthy[targetKey]) == 0 { + hc.healthy[targetKey] = append(hc.healthy[targetKey], hcc.tabletHealth) + } else { + // We already have one up server, see if we + // need to replace it. + if hcc.tabletHealth.MasterTermStartTime < hc.healthy[targetKey][0].MasterTermStartTime { + log.Warningf("not marking healthy master %s as Up for %s because its MasterTermStartTime is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", + topoproto.TabletAliasString(currentTablet.Alias), + topoproto.KeyspaceShardString(currentTablet.Keyspace, currentTablet.Shard), + topoproto.TabletAliasString(hc.healthy[targetKey][0].Tablet.Alias), + hcc.tabletHealth.MasterTermStartTime, + hc.healthy[targetKey][0].MasterTermStartTime) + } else { + // Just replace it. + hc.healthy[targetKey][0] = hcc.tabletHealth + } + } + } + // and notify downstream for master change + if shr.Target.TabletType == topodatapb.TabletType_MASTER { + if hc.masterCallback != nil { + hc.masterCallback(hcc.tabletHealth) + } + } return nil } @@ -579,17 +797,20 @@ func (hc *HealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { key := hc.keyFromTablet(tablet) tabletAlias := topoproto.TabletAliasString(tablet.Alias) - ths, ok := hc.healthData[key] + // delete from authoritative map + th, ok := hc.healthByAlias[tabletAlias] if !ok { - log.Warningf("Something is wrong, we have no health data for tablet: %v's target: %v", tabletAlias, key) + log.Warningf("Something is wrong, we have no health data for tablet: %v", tabletAlias) return } - th, ok := ths[tabletAlias] + th.deleteConnLocked() + delete(hc.healthByAlias, tabletAlias) + // delete from map by keyspace.shard.tabletType + ths, ok := hc.healthData[key] if !ok { - log.Warningf("Something is wrong, we have no health data for tablet: %v", tabletAlias) + log.Warningf("Something is wrong, we have no health data for target: %v", key) return } - th.deleteConnLocked() delete(ths, tabletAlias) } @@ -597,8 +818,9 @@ func (hc *HealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { // It does not block on making connection. // name is an optional tag for the tablet, e.g. an alternative address. func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet) { + log.Infof("Calling AddTablet for tablet: %v", tablet) hc.mu.Lock() - if hc.healthData == nil { + if hc.healthByAlias == nil { // already closed. hc.mu.Unlock() return @@ -615,20 +837,24 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet) { cancelFunc: cancelFunc, Tablet: tablet, Target: target, - Up: true, }, } // add to our datastore key := hc.keyFromTarget(target) tabletAlias := topoproto.TabletAliasString(tablet.Alias) + // TODO: can this ever already exist? + if _, ok := hc.healthByAlias[tabletAlias]; ok { + log.Errorf("Program bug") + return + } + hc.healthByAlias[tabletAlias] = hcc.tabletHealth if ths, ok := hc.healthData[key]; !ok { hc.healthData[key] = make(map[string]*TabletHealth) hc.healthData[key][tabletAlias] = hcc.tabletHealth } else { - if _, ok := ths[tabletAlias]; !ok { - ths[tabletAlias] = hcc.tabletHealth - } + // just overwrite it if it exists already? + ths[tabletAlias] = hcc.tabletHealth } hc.connsWG.Add(1) @@ -648,70 +874,29 @@ func (hc *HealthCheckImpl) ReplaceTablet(old, new *topodatapb.Tablet) { hc.AddTablet(new) } -// GetConnection returns the TabletConn of the given tablet. -func (hc *HealthCheckImpl) GetConnection(tabletAlias string) queryservice.QueryService { +// CacheStatus returns a displayable version of the cache. +func (hc *HealthCheckImpl) CacheStatus() TabletsCacheStatusList { + tcsl := make(TabletsCacheStatusList, 0, len(hc.healthByAlias)) hc.mu.Lock() defer hc.mu.Unlock() - - th := hc.findTabletHealthByAlias(tabletAlias) - if th == nil { - return nil - } - return th.Conn -} - -func (hc *HealthCheckImpl) findTabletHealthByAlias(alias string) *TabletHealth { - for _, ths := range hc.healthData { - for _, th := range ths { - if topoproto.TabletAliasString(th.Tablet.Alias) == alias { - return th - } + for _, th := range hc.healthByAlias { + tcs := &TabletsCacheStatus{ + Cell: th.Tablet.Alias.Cell, + Target: th.Target, } - } - return nil -} - -// CacheStatus returns a displayable version of the cache. -func (hc *HealthCheckImpl) CacheStatus() TabletsCacheStatusList { - tcsMap := hc.cacheStatusMap() - tcsl := make(TabletsCacheStatusList, 0, len(tcsMap)) - for _, tcs := range tcsMap { tcsl = append(tcsl, tcs) } sort.Sort(tcsl) return tcsl } -func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { - tcsMap := make(map[string]*TabletsCacheStatus) - hc.mu.Lock() - defer hc.mu.Unlock() - for _, ths := range hc.healthData { - for _, th := range ths { - key := fmt.Sprintf("%v.%v.%v.%v", th.Tablet.Alias.Cell, th.Target.Keyspace, th.Target.Shard, th.Target.TabletType.String()) - var tcs *TabletsCacheStatus - var ok bool - if tcs, ok = tcsMap[key]; !ok { - tcs = &TabletsCacheStatus{ - Cell: th.Tablet.Alias.Cell, - Target: th.Target, - } - tcsMap[key] = tcs - } - tcs.TabletsStats = append(tcs.TabletsStats, th) - } - } - return tcsMap -} - // Close stops the healthcheck. func (hc *HealthCheckImpl) Close() error { hc.mu.Lock() - for _, ths := range hc.healthData { - for _, th := range ths { - th.cancelFunc() - } + for _, th := range hc.healthByAlias { + th.cancelFunc() } + hc.healthByAlias = nil hc.healthData = nil for _, tw := range hc.topoWatchers { tw.Stop() @@ -768,8 +953,9 @@ func (hc *HealthCheckImpl) GetHealthyTabletStats(target *querypb.Target) []*Tabl log.Warningf("Can only have one master, program bug: %v", ths) return result } - for _, th := range ths { + for _, th := range hc.healthByAlias { if th.Tablet.Type == topodatapb.TabletType_MASTER { + // TODO(deepthi): return SimpleHealth here result = append(result, th.Copy()) return result } @@ -777,6 +963,8 @@ func (hc *HealthCheckImpl) GetHealthyTabletStats(target *querypb.Target) []*Tabl result = append(result, th.Copy()) } } + // healthy list needs to be sorted using replication lag algorithm + // so we might want to maintain it and update it instead of computing it here return result } @@ -786,10 +974,9 @@ func (hc *HealthCheckImpl) GetHealthyTabletStats(target *querypb.Target) []*Tabl // the most recent tablet of type master. func (hc *HealthCheckImpl) getTabletStats(target *querypb.Target) []*TabletHealth { var result []*TabletHealth - for _, ths := range hc.healthData { - for _, th := range ths { - result = append(result, th.Copy()) - } + ths := hc.healthData[hc.keyFromTarget(target)] + for _, th := range ths { + result = append(result, th.Copy()) } return result } diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go index d91c640bf1f..a7c9b68d59a 100644 --- a/go/vt/discovery/healthcheck_test.go +++ b/go/vt/discovery/healthcheck_test.go @@ -134,7 +134,6 @@ func TestTemplate(t *testing.T) { { Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, - Up: true, Serving: false, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3}, MasterTermStartTime: 0, @@ -165,7 +164,6 @@ func TestDebugURLFormatting(t *testing.T) { { Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, - Up: true, Serving: false, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3}, MasterTermStartTime: 0, @@ -200,5 +198,5 @@ func tabletDialer(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (quer } func createTestHc(ts *topo.Server) *HealthCheckImpl { - return NewHealthCheck(context.Background(), 1*time.Millisecond, time.Hour, ts, "cell").(*HealthCheckImpl) + return NewHealthCheck(context.Background(), 1*time.Millisecond, time.Hour, ts, "cell", nil).(*HealthCheckImpl) } diff --git a/go/vt/discovery/legacy_replicationlag.go b/go/vt/discovery/legacy_replicationlag.go index e7eedbbb2de..812f7c0dd07 100644 --- a/go/vt/discovery/legacy_replicationlag.go +++ b/go/vt/discovery/legacy_replicationlag.go @@ -91,7 +91,7 @@ func filterLegacyStatsByLag(tabletStatsList []*LegacyTabletStats) []*LegacyTable } // Sort by replication lag. - sort.Sort(byReplag(list)) + sort.Sort(byLegacyReplag(list)) // Pick those with low replication lag, but at least minNumTablets tablets regardless. res := make([]*LegacyTabletStats, 0, len(list)) @@ -129,10 +129,10 @@ func filterLegacyStatsByLagWithLegacyAlgorithm(tabletStatsList []*LegacyTabletSt // filter those affecting "mean" lag significantly // calculate mean for all tablets res := make([]*LegacyTabletStats, 0, len(list)) - m, _ := mean(list, -1) + m, _ := legacyMean(list, -1) for i, ts := range list { // calculate mean by excluding ith tablet - mi, _ := mean(list, i) + mi, _ := legacyMean(list, i) if float64(mi) > float64(m)*0.7 { res = append(res, ts) } @@ -167,7 +167,7 @@ func filterLegacyStatsByLagWithLegacyAlgorithm(tabletStatsList []*LegacyTabletSt } // Sort by replication lag. - sort.Sort(byReplag(snapshots)) + sort.Sort(byLegacyReplag(snapshots)) // Pick the first minNumTablets tablets. res = make([]*LegacyTabletStats, 0, *minNumTablets) @@ -181,15 +181,15 @@ type legacyTabletLagSnapshot struct { ts *LegacyTabletStats replag uint32 } -type byReplag []legacyTabletLagSnapshot +type byLegacyReplag []legacyTabletLagSnapshot -func (a byReplag) Len() int { return len(a) } -func (a byReplag) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a byReplag) Less(i, j int) bool { return a[i].replag < a[j].replag } +func (a byLegacyReplag) Len() int { return len(a) } +func (a byLegacyReplag) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byLegacyReplag) Less(i, j int) bool { return a[i].replag < a[j].replag } // mean calculates the mean value over the given list, // while excluding the item with the specified index. -func mean(tabletStatsList []*LegacyTabletStats, idxExclude int) (uint64, error) { +func legacyMean(tabletStatsList []*LegacyTabletStats, idxExclude int) (uint64, error) { var sum uint64 var count uint64 for i, ts := range tabletStatsList { diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index c47d6740ae8..b448d8ece7d 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -18,6 +18,7 @@ package discovery import ( "flag" + "fmt" "sort" "time" ) @@ -67,7 +68,17 @@ func IsReplicationLagVeryHigh(tabletHealth *TabletHealth) bool { // * degraded_threshold: this is only used by vttablet for display. It should match // discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. func FilterStatsByReplicationLag(tabletHealthList []*TabletHealth) []*TabletHealth { - return filterStatsByLag(tabletHealthList) + if !*legacyReplicationLagAlgorithm { + return filterStatsByLag(tabletHealthList) + } + res := filterStatsByLagWithLegacyAlgorithm(tabletHealthList) + // run the filter again if exactly one tablet is removed, + // and we have spare tablets. + if len(res) > *minNumTablets && len(res) == len(tabletHealthList)-1 { + res = filterStatsByLagWithLegacyAlgorithm(res) + } + return res + } func filterStatsByLag(tabletHealthList []*TabletHealth) []*TabletHealth { @@ -96,6 +107,86 @@ func filterStatsByLag(tabletHealthList []*TabletHealth) []*TabletHealth { return res } +func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*TabletHealth) []*TabletHealth { + list := make([]*TabletHealth, 0, len(tabletHealthList)) + // filter non-serving tablets + for _, ts := range tabletHealthList { + if !ts.Serving || ts.LastError != nil || ts.Stats == nil { + continue + } + list = append(list, ts) + } + if len(list) <= 1 { + return list + } + // if all have low replication lag (<=30s), return all tablets. + allLowLag := true + for _, ts := range list { + if IsReplicationLagHigh(ts) { + allLowLag = false + break + } + } + if allLowLag { + return list + } + // filter those affecting "mean" lag significantly + // calculate mean for all tablets + res := make([]*TabletHealth, 0, len(list)) + m, _ := mean(list, -1) + for i, ts := range list { + // calculate mean by excluding ith tablet + mi, _ := mean(list, i) + if float64(mi) > float64(m)*0.7 { + res = append(res, ts) + } + } + if len(res) >= *minNumTablets { + return res + } + // return at least minNumTablets tablets to avoid over loading, + // if there is enough tablets with replication lag < highReplicationLagMinServing. + // Pull the current replication lag for a stable sort. + snapshots := make([]tabletLagSnapshot, 0, len(list)) + for _, ts := range list { + if !IsReplicationLagVeryHigh(ts) { + snapshots = append(snapshots, tabletLagSnapshot{ + ts: ts, + replag: ts.Stats.SecondsBehindMaster}) + } + } + if len(snapshots) == 0 { + // We get here if all tablets are over the high + // replication lag threshold, and their lag is + // different enough that the 70% mean computation up + // there didn't find them all in a group. For + // instance, if *minNumTablets = 2, and we have two + // tablets with lag of 3h and 30h. In that case, we + // just use them all. + for _, ts := range list { + snapshots = append(snapshots, tabletLagSnapshot{ + ts: ts, + replag: ts.Stats.SecondsBehindMaster}) + } + } + + // Sort by replication lag. + sort.Sort(byReplag(snapshots)) + + // Pick the first minNumTablets tablets. + res = make([]*TabletHealth, 0, *minNumTablets) + for i := 0; i < min(*minNumTablets, len(snapshots)); i++ { + res = append(res, snapshots[i].ts) + } + return res +} + +type byReplag []tabletLagSnapshot + +func (a byReplag) Len() int { return len(a) } +func (a byReplag) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byReplag) Less(i, j int) bool { return a[i].replag < a[j].replag } + type tabletLagSnapshot struct { ts *TabletHealth replag uint32 @@ -112,3 +203,21 @@ func min(a, b int) int { } return a } + +// mean calculates the mean value over the given list, +// while excluding the item with the specified index. +func mean(tabletHealthList []*TabletHealth, idxExclude int) (uint64, error) { + var sum uint64 + var count uint64 + for i, ts := range tabletHealthList { + if i == idxExclude { + continue + } + sum = sum + uint64(ts.Stats.SecondsBehindMaster) + count++ + } + if count == 0 { + return 0, fmt.Errorf("empty list") + } + return sum / count, nil +} diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 27ac8911274..66d3ff0b01f 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -29,8 +29,6 @@ type TabletHealth struct { // Target is the current target as returned by the streaming // StreamHealth RPC. Target *query.Target - // Up describes whether the tablet is added or removed. - Up bool // Serving describes if the tablet can be serving traffic. Serving bool // MasterTermStartTime is the last time at which @@ -50,8 +48,8 @@ type TabletHealth struct { func (th *TabletHealth) String() string { th.mu.Lock() defer th.mu.Unlock() - return fmt.Sprintf("TabletHealth{Tablet: %v,Target: %v,Up: %v,Serving: %v, MasterTermStartTime: %v, Stats: %v, LastError: %v", - th.Tablet, th.Target, th.Up, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) + return fmt.Sprintf("TabletHealth{Tablet: %v,Target: %v,Serving: %v, MasterTermStartTime: %v, Stats: %v, LastError: %v", + th.Tablet, th.Target, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) } // Copy returns a copy of TabletHealth. Note that this is not really a deep copy @@ -68,7 +66,6 @@ func (th *TabletHealth) Copy() *TabletHealth { Conn: th.Conn, Tablet: th.Tablet, Target: th.Target, - Up: th.Up, Serving: th.Serving, MasterTermStartTime: th.MasterTermStartTime, Stats: th.Stats, @@ -81,7 +78,6 @@ func (th *TabletHealth) Copy() *TabletHealth { func (th *TabletHealth) DeepEqual(other *TabletHealth) bool { return proto.Equal(th.Tablet, other.Tablet) && proto.Equal(th.Target, other.Target) && - th.Up == other.Up && th.Serving == other.Serving && th.MasterTermStartTime == other.MasterTermStartTime && proto.Equal(th.Stats, other.Stats) && @@ -135,7 +131,6 @@ func (th *TabletHealth) getTabletDebugURL() string { func (th *TabletHealth) deleteConnLocked() { th.mu.Lock() - th.Up = false th.Conn = nil th.mu.Unlock() th.cancelFunc() diff --git a/go/vt/discovery/tablets_cache_status.go b/go/vt/discovery/tablets_cache_status.go index 6b0a1d4efb6..16335f679df 100644 --- a/go/vt/discovery/tablets_cache_status.go +++ b/go/vt/discovery/tablets_cache_status.go @@ -53,9 +53,6 @@ func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML { } else if !ts.Serving { color = "red" extra = " (Not Serving)" - } else if !ts.Up { - color = "red" - extra = " (Down)" } else if ts.Target.TabletType == topodatapb.TabletType_MASTER { extra = fmt.Sprintf(" (MasterTS: %v)", ts.MasterTermStartTime) } else { diff --git a/go/vt/discovery/topology_watcher.go b/go/vt/discovery/topology_watcher.go index 396389c21b9..9b01befd1bb 100644 --- a/go/vt/discovery/topology_watcher.go +++ b/go/vt/discovery/topology_watcher.go @@ -17,16 +17,11 @@ limitations under the License. package discovery import ( - "bytes" "fmt" - "hash/crc32" - "sort" "strings" "sync" "time" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/key" "golang.org/x/net/context" @@ -59,21 +54,12 @@ type tabletInfo struct { tablet *topodatapb.Tablet } -// NewCellTabletsWatcher returns a TopologyWatcher that monitors all -// the tablets in a cell, and starts refreshing. -func NewCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, f TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *TopologyWatcher { - return NewTopologyWatcher(ctx, topoServer, tr, f, cell, refreshInterval, refreshKnownTablets, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) { - return tw.topoServer.GetTabletsByCell(ctx, tw.cell) - }) -} - // TopologyWatcher polls tablet from a configurable set of tablets // periodically. When tablets are added / removed, it calls // the LegacyTabletRecorder AddTablet / RemoveTablet interface appropriately. type TopologyWatcher struct { // set at construction time topoServer *topo.Server - tabletRecorder TabletRecorder tabletFilter TabletFilter cell string refreshInterval time.Duration @@ -97,16 +83,13 @@ type TopologyWatcher struct { firstLoadDone bool // firstLoadChan is closed when the initial loading of topology data is done. firstLoadChan chan struct{} - // cellAliases is a cache of cell aliases - cellAliases map[string]string } // NewTopologyWatcher returns a TopologyWatcher that monitors all // the tablets in a cell, and starts refreshing. -func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, filter TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error)) *TopologyWatcher { +func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, filter TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error)) *TopologyWatcher { tw := &TopologyWatcher{ topoServer: topoServer, - tabletRecorder: tr, tabletFilter: filter, cell: cell, refreshInterval: refreshInterval, @@ -114,7 +97,6 @@ func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr TabletR getTablets: getTablets, sem: make(chan int, topoReadConcurrency), tablets: make(map[string]*tabletInfo), - cellAliases: make(map[string]string), } tw.firstLoadChan = make(chan struct{}) @@ -124,6 +106,14 @@ func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr TabletR return tw } +// NewCellTabletsWatcher returns a TopologyWatcher that monitors all +// the tablets in a cell, and starts refreshing. +func NewCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, f TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *TopologyWatcher { + return NewTopologyWatcher(ctx, topoServer, f, cell, refreshInterval, refreshKnownTablets, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) { + return tw.topoServer.GetTabletsByCell(ctx, tw.cell) + }) +} + // WaitForInitialTopology waits until the watcher reads all of the topology data // for the first time and transfers the information to LegacyTabletRecorder via its // AddTablet() method. @@ -159,175 +149,6 @@ func (tw *TopologyWatcher) TopoChecksum() uint32 { return tw.topoChecksum } -func (tw *TopologyWatcher) watchTopo() { - tw.wg.Add(1) - defer tw.wg.Done() - ticker := time.NewTicker(tw.refreshInterval) - defer ticker.Stop() - for { - tw.loadTablets() - select { - case <-tw.ctx.Done(): - return - case <-ticker.C: - } - } -} - -func (tw *TopologyWatcher) loadTablets() { - var wg sync.WaitGroup - newTablets := make(map[string]*tabletInfo) - replacedTablets := make(map[string]*tabletInfo) - - tabletAliases, err := tw.getTablets(tw) - topologyWatcherOperations.Add(topologyWatcherOpListTablets, 1) - if err != nil { - topologyWatcherErrors.Add(topologyWatcherOpListTablets, 1) - select { - case <-tw.ctx.Done(): - return - default: - } - log.Errorf("cannot get tablets for cell: %v: %v", tw.cell, err) - return - } - - // Accumulate a list of all known alias strings to use later - // when sorting - tabletAliasStrs := make([]string, 0, len(tabletAliases)) - - tw.mu.Lock() - for _, tAlias := range tabletAliases { - aliasStr := topoproto.TabletAliasString(tAlias) - tabletAliasStrs = append(tabletAliasStrs, aliasStr) - - if !tw.refreshKnownTablets { - if val, ok := tw.tablets[aliasStr]; ok { - newTablets[aliasStr] = val - continue - } - } - - wg.Add(1) - go func(alias *topodatapb.TabletAlias) { - defer wg.Done() - tw.sem <- 1 // Wait for active queue to drain. - tablet, err := tw.topoServer.GetTablet(tw.ctx, alias) - topologyWatcherOperations.Add(topologyWatcherOpGetTablet, 1) - <-tw.sem // Done; enable next request to run - if err != nil { - topologyWatcherErrors.Add(topologyWatcherOpGetTablet, 1) - select { - case <-tw.ctx.Done(): - return - default: - } - log.Errorf("cannot get tablet for alias %v: %v", alias, err) - return - } - if !(tw.isTabletInCell(tablet.Tablet) && (tw.tabletFilter == nil || tw.tabletFilter.IsIncluded(tablet.Tablet))) { - return - } - tw.mu.Lock() - aliasStr := topoproto.TabletAliasString(alias) - newTablets[aliasStr] = &tabletInfo{ - alias: aliasStr, - tablet: tablet.Tablet, - } - tw.mu.Unlock() - }(tAlias) - } - - tw.mu.Unlock() - wg.Wait() - tw.mu.Lock() - - for alias, newVal := range newTablets { - if val, ok := tw.tablets[alias]; !ok { - // Check if there's a tablet with the same address key but a - // different alias. If so, replace it and keep track of the - // replaced alias to make sure it isn't removed later. - found := false - for _, otherVal := range tw.tablets { - if newVal.alias == otherVal.alias { - found = true - tw.tabletRecorder.ReplaceTablet(otherVal.tablet, newVal.tablet) - topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) - replacedTablets[otherVal.alias] = newVal - } - } - if !found { - tw.tabletRecorder.AddTablet(newVal.tablet) - topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1) - } - - } else if val.alias != newVal.alias { - // Handle the case where the same tablet alias is now reporting - // a different address key. - replacedTablets[alias] = newVal - tw.tabletRecorder.ReplaceTablet(val.tablet, newVal.tablet) - topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) - } - } - - for _, val := range tw.tablets { - if _, ok := newTablets[val.alias]; !ok { - if _, ok2 := replacedTablets[val.alias]; !ok2 { - tw.tabletRecorder.RemoveTablet(val.tablet) - topologyWatcherOperations.Add(topologyWatcherOpRemoveTablet, 1) - } - } - } - tw.tablets = newTablets - if !tw.firstLoadDone { - tw.firstLoadDone = true - close(tw.firstLoadChan) - } - - // iterate through the tablets in a stable order and compute a - // checksum of the tablet map - sort.Strings(tabletAliasStrs) - var buf bytes.Buffer - for _, alias := range tabletAliasStrs { - _, ok := tw.tablets[alias] - if ok { - buf.WriteString(alias) - } - } - tw.topoChecksum = crc32.ChecksumIEEE(buf.Bytes()) - tw.lastRefresh = time.Now() - - tw.mu.Unlock() - -} - -func (tw *TopologyWatcher) getAliasByCell(cell string) string { - tw.mu.Lock() - defer tw.mu.Unlock() - - if alias, ok := tw.cellAliases[cell]; ok { - return alias - } - - alias := topo.GetAliasByCell(context.Background(), tw.topoServer, cell) - tw.cellAliases[cell] = alias - - return alias -} - -func (tw *TopologyWatcher) isTabletInCell(tablet *topodatapb.Tablet) bool { - if tablet.Type == topodatapb.TabletType_MASTER { - return true - } - if tablet.Alias.Cell == tw.cell { - return true - } - if tw.getAliasByCell(tablet.Alias.Cell) == tw.getAliasByCell(tw.cell) { - return true - } - return false -} - // TabletFilter is an interface that can be given to a TopologyWatcher // to be applied as an additional filter on the list of tablets returned by its getTablets function type TabletFilter interface { diff --git a/go/vt/servenv/status.go b/go/vt/servenv/status.go index 619e75f0956..1d41c1acbb5 100644 --- a/go/vt/servenv/status.go +++ b/go/vt/servenv/status.go @@ -223,6 +223,8 @@ func (sp *statusPage) statusHandler(w http.ResponseWriter, r *http.Request) { if err := sp.tmpl.ExecuteTemplate(w, "status", data); err != nil { if _, ok := err.(net.Error); !ok { log.Errorf("servenv: couldn't execute template: %v", err) + log.Infof("template: %v", sp.tmpl) + log.Infof("data: %v", data) } } } diff --git a/go/vt/vtgate/buffer/buffer.go b/go/vt/vtgate/buffer/buffer.go index 0327683d4a6..0568d6df950 100644 --- a/go/vt/vtgate/buffer/buffer.go +++ b/go/vt/vtgate/buffer/buffer.go @@ -32,8 +32,6 @@ import ( "sync" "time" - querypb "vitess.io/vitess/go/vt/proto/query" - "golang.org/x/net/context" "vitess.io/vitess/go/sync2" @@ -215,32 +213,17 @@ func (b *Buffer) WaitForFailoverEnd(ctx context.Context, keyspace, shard string, return sb.waitForFailoverEnd(ctx, keyspace, shard, err) } -// WaitingForFailoverEnd tells us whether we are currently buffering -// which means someone might be waiting for the failover to end -func (b *Buffer) WaitingForFailoverEnd(ctx context.Context, keyspace, shard string) bool { - - sb := b.getOrCreateBuffer(keyspace, shard) - if sb == nil { - // buffer is stopped - return false - } - if sb.disabled() { - // no buffering is enabled so nothing to do - return false +// NewMasterDetected notifies the buffer to record a new master +// and end any failover buffering that may be in progress +func (b *Buffer) NewMasterDetected(th *discovery.TabletHealth) { + timestamp := th.MasterTermStartTime + if timestamp == 0 { + // Masters where TabletExternallyReparented was never called will return 0. + // Ignore them. + return } - // only returns true if sb.state == stateBuffering - return sb.shouldBufferLocked(false) -} - -// EndFailover tells the buffer that the failover has ended -// so it can start retrying the buffered requests -func (b *Buffer) EndFailover(target *querypb.Target, th *discovery.TabletHealth) { - if target.TabletType != topodatapb.TabletType_MASTER { - panic(fmt.Sprintf("BUG: non MASTER target cannot end failover: %v", target)) - } - timestamp := th.MasterTermStartTime - sb := b.getOrCreateBuffer(target.Keyspace, target.Shard) + sb := b.getOrCreateBuffer(th.Target.Keyspace, th.Target.Shard) if sb == nil { // Buffer is shut down. Ignore all calls. return diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index a705af537d9..b3609f8dc63 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -82,7 +82,10 @@ func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string log.Exitf("Unable to create new TabletGateway: %v", err) } } - hc := discovery.NewHealthCheck(ctx, *HealthCheckRetryDelay, *HealthCheckTimeout, topoServer, localCell) + b := buffer.New() + // callback to notify buffer when to end failover + newMasterDetected := b.NewMasterDetected + hc := discovery.NewHealthCheck(ctx, *HealthCheckRetryDelay, *HealthCheckTimeout, topoServer, localCell, newMasterDetected) gw := &TabletGateway{ hc: hc, @@ -90,7 +93,7 @@ func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string localCell: localCell, retryCount: *RetryCount, statusAggregators: make(map[string]*TabletStatusAggregator), - buffer: buffer.New(), + buffer: b, } gw.QueryService = queryservice.Wrap(nil, gw.withRetry) return gw @@ -189,6 +192,8 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, } } + // instead of returning []*TabletHealth we can return a simpler struct + // that contains just tablet ALias and connection tablets := gw.hc.GetHealthyTabletStats(target) if len(tablets) == 0 { // fail fast if there is no tablet @@ -228,13 +233,6 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, var canRetry bool canRetry, err = inner(ctx, target, th.Conn) gw.updateStats(target, startTime, err) - - // if a master query succeeded and the buffer is currently buffering, end the buffering - if err == nil && target.TabletType == topodatapb.TabletType_MASTER && gw.buffer.WaitingForFailoverEnd(ctx, target.Keyspace, target.Shard) { - // notify buffer that failover has ended - gw.buffer.EndFailover(target, th) - } - if canRetry { invalidTablets[tabletLastUsed] = true continue From 1c511af4ee32824385ec65ce19a3621ff11c5f71 Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 4 May 2020 21:31:50 -0700 Subject: [PATCH 17/39] healthcheck: fold healthCheckConn into TabletHealth, fix CacheStatus, get basic unit test working Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 329 +++++++----------------- go/vt/discovery/healthcheck_test.go | 218 ++++++++++++++-- go/vt/discovery/tablet_health.go | 178 +++++++++++++ go/vt/discovery/tablets_cache_status.go | 34 +++ 4 files changed, 496 insertions(+), 263 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index f417d4ee844..1972805372c 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -49,17 +49,12 @@ import ( "vitess.io/vitess/go/vt/topo" - "github.com/golang/protobuf/proto" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/sync2" - "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/topotools" - "vitess.io/vitess/go/vt/vttablet/tabletconn" - - querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) var ( @@ -76,7 +71,7 @@ var ( // CellsToWatch is the list of cells the healthcheck operates over. If it is empty, only the local cell is watched CellsToWatch = flag.String("cells_to_watch", "", "comma-separated list of cells for watching tablets") // AllowedTabletTypes is the list of allowed tablet types. e.g. {MASTER, REPLICA} - AllowedTabletTypes []topodatapb.TabletType + AllowedTabletTypes []topodata.TabletType // TabletFilters are the keyspace|shard or keyrange filters to apply to the full set of tablets TabletFilters flagutil.StringListValue // KeyspacesToWatch - if provided this specifies which keyspaces should be @@ -166,15 +161,15 @@ type HealthCheck interface { // Close stops the healthcheck. Close() error // GetHealthyTabletStatts - GetHealthyTabletStats(target *querypb.Target) []*TabletHealth + GetHealthyTabletStats(target *query.Target) []*TabletHealth // WaitForAllServingTablets allows vtgate to wait for all tablets to be serving before accepting requests - WaitForAllServingTablets(ctx context.Context, targets []*querypb.Target) error + WaitForAllServingTablets(ctx context.Context, targets []*query.Target) error // AddTablet adds the tablet. - AddTablet(tablet *topodatapb.Tablet) + AddTablet(tablet *topodata.Tablet) // RemoveTablet removes the tablet. - RemoveTablet(tablet *topodatapb.Tablet) + RemoveTablet(tablet *topodata.Tablet) // ReplaceTablet does an AddTablet and RemoveTablet in one call, effectively replacing the old tablet with the new. - ReplaceTablet(old, new *topodatapb.Tablet) + ReplaceTablet(old, new *topodata.Tablet) } // HealthCheckImpl performs health checking and stores the results. @@ -218,19 +213,6 @@ type HealthCheckImpl struct { // Conn queryservice.QueryService //} -// HealthCheckConn is a structure that lives within the scope of -// the checkConn goroutine to maintain its internal state. Therefore, -// it does not require synchronization. Changes that are relevant to -// healthcheck are transmitted through changes to the TabletHealth -// object, which has its own mutex. -type healthCheckConn struct { - ctx context.Context - - tabletHealth *TabletHealth - loggedServingState bool - lastResponseTimestamp time.Time // timestamp of the last healthcheck response -} - // NewHealthCheck creates a new HealthCheck object. // Parameters: // retryDelay. @@ -348,7 +330,7 @@ func (hc *HealthCheckImpl) loadTablets(tw *TopologyWatcher) { } wg.Add(1) - go func(alias *topodatapb.TabletAlias) { + go func(alias *topodata.TabletAlias) { defer wg.Done() tw.sem <- 1 // Wait for active queue to drain. tablet, err := tw.topoServer.GetTablet(tw.ctx, alias) @@ -443,8 +425,8 @@ func (hc *HealthCheckImpl) getAliasByCell(cell string) string { return alias } -func (hc *HealthCheckImpl) isTabletInCell(tablet *topodatapb.Tablet) bool { - if tablet.Type == topodatapb.TabletType_MASTER { +func (hc *HealthCheckImpl) isTabletInCell(tablet *topodata.Tablet) bool { + if tablet.Type == topodata.TabletType_MASTER { return true } if tablet.Alias.Cell == hc.cell { @@ -471,7 +453,7 @@ func (hc *HealthCheckImpl) RegisterStats() { ) stats.NewGaugesFuncWithMultiLabels( - "HealthcheckConnections", + "TabletHealthections", "the number of healthcheck connections registered", []string{"Keyspace", "ShardName", "TabletType"}, hc.servingConnStats) @@ -538,31 +520,31 @@ func (hc *HealthCheckImpl) stateChecksum() int64 { // finalizeConn closes the health checking connection and sends the final // notification about the tablet to downstream. To be called only on exit from // checkConn(). -func (hc *HealthCheckImpl) finalizeConn(hcc *healthCheckConn) { - hcc.tabletHealth.mu.Lock() - defer hcc.tabletHealth.mu.Unlock() - hcc.setServingState(false, "finalizeConn closing connection") - // Note: checkConn() exits only when hcc.ctx.Done() is closed. Thus it's +func (hc *HealthCheckImpl) finalizeConn(th *TabletHealth) { + th.mu.Lock() + defer th.mu.Unlock() + th.setServingState(false, "finalizeConn closing connection") + // Note: checkConn() exits only when th.ctx.Done() is closed. Thus it's // safe to simply get Err() value here and assign to LastError. - hcc.tabletHealth.LastError = hcc.ctx.Err() - if hcc.tabletHealth.Conn != nil { - // Don't use hcc.ctx because it's already closed. + th.LastError = th.ctx.Err() + if th.Conn != nil { + // Don't use th.ctx because it's already closed. // Use a separate context, and add a timeout to prevent unbounded waits. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - hcc.tabletHealth.Conn.Close(ctx) - hcc.tabletHealth.Conn = nil + th.Conn.Close(ctx) + th.Conn = nil } } // checkConn performs health checking on the given tablet. -func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn) { +func (hc *HealthCheckImpl) checkConn(th *TabletHealth) { defer hc.connsWG.Done() - defer hc.finalizeConn(hcc) + defer hc.finalizeConn(th) retryDelay := hc.retryDelay for { - streamCtx, streamCancel := context.WithCancel(hcc.ctx) + streamCtx, streamCancel := context.WithCancel(th.ctx) // Setup a watcher that restarts the timer every time an update is received. // If a timeout occurs for a serving tablet, we make it non-serving and send @@ -591,7 +573,7 @@ func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn) { }() // Read stream health responses. - hcc.stream(streamCtx, func(shr *querypb.StreamHealthResponse) error { + err := th.stream(streamCtx, func(shr *query.StreamHealthResponse) error { // We received a message. Reset the back-off. retryDelay = hc.retryDelay // Don't block on send to avoid deadlocks. @@ -599,27 +581,32 @@ func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn) { case servingStatus <- shr.Serving: default: } - return hcc.processResponse(hc, shr) + return th.processResponse(hc, shr) }) // streamCancel to make sure the watcher goroutine terminates. streamCancel() + if err != nil && strings.Contains(err.Error(), "health stats mismatch") { + //finalizeConn will delete all data once this loop breaks + return + } // If there was a timeout send an error. We do this after stream has returned. // This will ensure that this update prevails over any previous message that // stream could have sent. if timedout.Get() { - hcc.tabletHealth.mu.Lock() - hcc.tabletHealth.LastError = fmt.Errorf("healthcheck timed out (latest %v)", hcc.lastResponseTimestamp) - hcc.setServingState(false, hcc.tabletHealth.LastError.Error()) - hcErrorCounters.Add([]string{hcc.tabletHealth.Target.Keyspace, hcc.tabletHealth.Target.Shard, topoproto.TabletTypeLString(hcc.tabletHealth.Target.TabletType)}, 1) - hcc.tabletHealth.mu.Unlock() + th.mu.Lock() + // get timestamp from error and put it into LastError and remove th.lastResponseTimestamp + th.LastError = fmt.Errorf("healthcheck timed out (latest %v)", th.lastResponseTimestamp) + th.setServingState(false, th.LastError.Error()) + hcErrorCounters.Add([]string{th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)}, 1) + th.mu.Unlock() } // Streaming RPC failed e.g. because vttablet was restarted or took too long. // Sleep until the next retry is up or the context is done/canceled. select { - case <-hcc.ctx.Done(): + case <-th.ctx.Done(): return case <-time.After(retryDelay): // Exponentially back-off to prevent tight-loop. @@ -632,166 +619,7 @@ func (hc *HealthCheckImpl) checkConn(hcc *healthCheckConn) { } } -// setServingState sets the tablet state to the given value. -// -// If the state changes, it logs the change so that failures -// from the health check connection are logged the first time, -// but don't continue to log if the connection stays down. -// -// hcc.TabletHealth.mu must be locked before calling this function -func (hcc *healthCheckConn) setServingState(serving bool, reason string) { - if !hcc.loggedServingState || (serving != hcc.tabletHealth.Serving) { - // Emit the log from a separate goroutine to avoid holding - // the hcc lock while logging is happening - go log.Infof("HealthCheckUpdate(Serving State): tablet: %v serving => %v for %v/%v (%v) reason: %s", - topotools.TabletIdent(hcc.tabletHealth.Tablet), - serving, - hcc.tabletHealth.Tablet.GetKeyspace(), - hcc.tabletHealth.Tablet.GetShard(), - hcc.tabletHealth.Target.GetTabletType(), - reason, - ) - hcc.loggedServingState = true - } - hcc.tabletHealth.Serving = serving -} - -// stream streams healthcheck responses to callback. -func (hcc *healthCheckConn) stream(ctx context.Context, callback func(*querypb.StreamHealthResponse) error) { - hcc.tabletHealth.mu.Lock() - if hcc.tabletHealth.Conn == nil { - conn, err := tabletconn.GetDialer()(hcc.tabletHealth.Tablet, grpcclient.FailFast(true)) - if err != nil { - hcc.tabletHealth.LastError = err - hcc.tabletHealth.mu.Unlock() - return - } - hcc.tabletHealth.Conn = conn - hcc.tabletHealth.LastError = nil - } - conn := hcc.tabletHealth.Conn - hcc.tabletHealth.mu.Unlock() - - if err := conn.StreamHealth(ctx, callback); err != nil { - hcc.tabletHealth.mu.Lock() - log.Warningf("tablet %v healthcheck stream error: %v", hcc.tabletHealth.Tablet.Alias, err) - hcc.setServingState(false, err.Error()) - hcc.tabletHealth.LastError = err - hcc.tabletHealth.Conn.Close(ctx) - hcc.tabletHealth.Conn = nil - hcc.tabletHealth.mu.Unlock() - } -} - -// processResponse reads one health check response, and updates health -func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, shr *querypb.StreamHealthResponse) error { - select { - case <-hcc.ctx.Done(): - return hcc.ctx.Err() - default: - } - - // Check for invalid data, better than panicking. - if shr.Target == nil || shr.RealtimeStats == nil { - return fmt.Errorf("health stats is not valid: %v", shr) - } - - // an app-level error from tablet, force serving state. - var healthErr error - serving := shr.Serving - if shr.RealtimeStats.HealthError != "" { - healthErr = fmt.Errorf("vttablet error: %v", shr.RealtimeStats.HealthError) - serving = false - } - - if shr.TabletAlias != nil && !proto.Equal(shr.TabletAlias, hcc.tabletHealth.Tablet.Alias) { - // TabletAlias change means that the host:port has been taken over by another tablet - // We could cancel / exit the healthcheck for this tablet right away - // However, we defer it until the next topo refresh informs us of the change because that is - // the only way to discover the new host/port - return fmt.Errorf("health stats mismatch, tablet %+v alias does not match response alias %v", hcc.tabletHealth.Tablet, shr.TabletAlias) - } - - hcc.tabletHealth.mu.Lock() - currentTablet := hcc.tabletHealth.Tablet - // check whether this is a trivial update so as to update healthy map - trivialNonMasterUpdate := hcc.tabletHealth.LastError == nil && hcc.tabletHealth.Serving && shr.RealtimeStats.HealthError == "" && shr.Serving && - currentTablet.Type != topodatapb.TabletType_MASTER && currentTablet.Type == shr.Target.TabletType - isMasterUpdate := currentTablet.Type == topodatapb.TabletType_MASTER && shr.Target.TabletType == topodatapb.TabletType_MASTER - hcc.tabletHealth.mu.Unlock() - - // hc.healthByAlias is authoritative, it should be updated - hc.mu.Lock() - tabletAlias := topoproto.TabletAliasString(hcc.tabletHealth.Tablet.Alias) - // this will only change the first time, but it's easiest to set it always rather than check and set - hc.healthByAlias[tabletAlias] = hcc.tabletHealth - hc.mu.Unlock() - // In this case where a new tablet is initialized or a tablet type changes, we want to - // initialize the counter so the rate can be calculated correctly. - if currentTablet.Type != shr.Target.TabletType || currentTablet.Keyspace != shr.Target.Keyspace || currentTablet.Shard != shr.Target.Shard { - hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) - // hc still has this TabletHealth in the wrong target (because tabletType changed) - oldTargetKey := hc.keyFromTablet(currentTablet) - newTargetKey := hc.keyFromTarget(shr.Target) - tabletAlias := topoproto.TabletAliasString(currentTablet.Alias) - hc.mu.Lock() - delete(hc.healthData[oldTargetKey], tabletAlias) - hc.healthData[newTargetKey][tabletAlias] = hcc.tabletHealth - hc.mu.Unlock() - } - - // Update our record - hcc.lastResponseTimestamp = time.Now() - hcc.tabletHealth.mu.Lock() - defer hcc.tabletHealth.mu.Unlock() - hcc.tabletHealth.Target = shr.Target - hcc.tabletHealth.MasterTermStartTime = shr.TabletExternallyReparentedTimestamp - hcc.tabletHealth.Stats = shr.RealtimeStats - hcc.tabletHealth.LastError = healthErr - reason := "healthCheck update" - if healthErr != nil { - reason = "healthCheck update error: " + healthErr.Error() - } - hcc.setServingState(serving, reason) - - targetKey := hc.keyFromTarget(shr.Target) - if !trivialNonMasterUpdate { - all := hc.healthData[targetKey] - allArray := make([]*TabletHealth, 0, len(all)) - for _, s := range all { - allArray = append(allArray, s) - } - hc.healthy[targetKey] = FilterStatsByReplicationLag(allArray) - } - if isMasterUpdate { - if len(hc.healthy[targetKey]) == 0 { - hc.healthy[targetKey] = append(hc.healthy[targetKey], hcc.tabletHealth) - } else { - // We already have one up server, see if we - // need to replace it. - if hcc.tabletHealth.MasterTermStartTime < hc.healthy[targetKey][0].MasterTermStartTime { - log.Warningf("not marking healthy master %s as Up for %s because its MasterTermStartTime is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", - topoproto.TabletAliasString(currentTablet.Alias), - topoproto.KeyspaceShardString(currentTablet.Keyspace, currentTablet.Shard), - topoproto.TabletAliasString(hc.healthy[targetKey][0].Tablet.Alias), - hcc.tabletHealth.MasterTermStartTime, - hc.healthy[targetKey][0].MasterTermStartTime) - } else { - // Just replace it. - hc.healthy[targetKey][0] = hcc.tabletHealth - } - } - } - // and notify downstream for master change - if shr.Target.TabletType == topodatapb.TabletType_MASTER { - if hc.masterCallback != nil { - hc.masterCallback(hcc.tabletHealth) - } - } - return nil -} - -func (hc *HealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { +func (hc *HealthCheckImpl) deleteConn(tablet *topodata.Tablet) { hc.mu.Lock() defer hc.mu.Unlock() @@ -817,7 +645,7 @@ func (hc *HealthCheckImpl) deleteConn(tablet *topodatapb.Tablet) { // AddTablet adds the tablet, and starts health check. // It does not block on making connection. // name is an optional tag for the tablet, e.g. an alternative address. -func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet) { +func (hc *HealthCheckImpl) AddTablet(tablet *topodata.Tablet) { log.Infof("Calling AddTablet for tablet: %v", tablet) hc.mu.Lock() if hc.healthByAlias == nil { @@ -826,18 +654,16 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet) { return } ctx, cancelFunc := context.WithCancel(context.Background()) - target := &querypb.Target{ + target := &query.Target{ Keyspace: tablet.Keyspace, Shard: tablet.Shard, TabletType: tablet.Type, } - hcc := &healthCheckConn{ - ctx: ctx, - tabletHealth: &TabletHealth{ - cancelFunc: cancelFunc, - Tablet: tablet, - Target: target, - }, + th := &TabletHealth{ + ctx: ctx, + cancelFunc: cancelFunc, + Tablet: tablet, + Target: target, } // add to our datastore @@ -848,46 +674,63 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodatapb.Tablet) { log.Errorf("Program bug") return } - hc.healthByAlias[tabletAlias] = hcc.tabletHealth + hc.healthByAlias[tabletAlias] = th if ths, ok := hc.healthData[key]; !ok { hc.healthData[key] = make(map[string]*TabletHealth) - hc.healthData[key][tabletAlias] = hcc.tabletHealth + hc.healthData[key][tabletAlias] = th } else { // just overwrite it if it exists already? - ths[tabletAlias] = hcc.tabletHealth + ths[tabletAlias] = th } hc.connsWG.Add(1) hc.mu.Unlock() - go hc.checkConn(hcc) + go hc.checkConn(th) } // RemoveTablet removes the tablet, and stops the health check. // It does not block. -func (hc *HealthCheckImpl) RemoveTablet(tablet *topodatapb.Tablet) { +func (hc *HealthCheckImpl) RemoveTablet(tablet *topodata.Tablet) { hc.deleteConn(tablet) } // ReplaceTablet removes the old tablet and adds the new tablet. -func (hc *HealthCheckImpl) ReplaceTablet(old, new *topodatapb.Tablet) { +func (hc *HealthCheckImpl) ReplaceTablet(old, new *topodata.Tablet) { hc.deleteConn(old) hc.AddTablet(new) } // CacheStatus returns a displayable version of the cache. func (hc *HealthCheckImpl) CacheStatus() TabletsCacheStatusList { - tcsl := make(TabletsCacheStatusList, 0, len(hc.healthByAlias)) + tcsMap := hc.cacheStatusMap() + tcsl := make(TabletsCacheStatusList, 0, len(tcsMap)) + for _, tcs := range tcsMap { + tcsl = append(tcsl, tcs) + } + sort.Sort(tcsl) + return tcsl +} + +func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { + tcsMap := make(map[string]*TabletsCacheStatus) hc.mu.Lock() defer hc.mu.Unlock() for _, th := range hc.healthByAlias { - tcs := &TabletsCacheStatus{ - Cell: th.Tablet.Alias.Cell, - Target: th.Target, + key := fmt.Sprintf("%v.%v.%v.%v", th.Tablet.Alias.Cell, th.Target.Keyspace, th.Target.Shard, th.Target.TabletType.String()) + var tcs *TabletsCacheStatus + var ok bool + th.mu.Lock() + if tcs, ok = tcsMap[key]; !ok { + tcs = &TabletsCacheStatus{ + Cell: th.Tablet.Alias.Cell, + Target: th.Target, + } + tcsMap[key] = tcs } - tcsl = append(tcsl, tcs) + tcs.TabletsStats = append(tcs.TabletsStats, th) + th.mu.Unlock() } - sort.Sort(tcsl) - return tcsl + return tcsMap } // Close stops the healthcheck. @@ -940,7 +783,7 @@ func (hc *HealthCheckImpl) topologyWatcherChecksum() int64 { // the most recent tablet of type master. // This returns a copy of the data so that callers can access without // synchronization -func (hc *HealthCheckImpl) GetHealthyTabletStats(target *querypb.Target) []*TabletHealth { +func (hc *HealthCheckImpl) GetHealthyTabletStats(target *query.Target) []*TabletHealth { var result []*TabletHealth // we check all tablet types in all cells because of cellAliases key := hc.keyFromTarget(target) @@ -949,12 +792,12 @@ func (hc *HealthCheckImpl) GetHealthyTabletStats(target *querypb.Target) []*Tabl log.Warningf("Healthcheck has no tablet health for target: %v", key) return result } - if target.TabletType == topodatapb.TabletType_MASTER && len(ths) > 1 { + if target.TabletType == topodata.TabletType_MASTER && len(ths) > 1 { log.Warningf("Can only have one master, program bug: %v", ths) return result } for _, th := range hc.healthByAlias { - if th.Tablet.Type == topodatapb.TabletType_MASTER { + if th.Tablet.Type == topodata.TabletType_MASTER { // TODO(deepthi): return SimpleHealth here result = append(result, th.Copy()) return result @@ -972,7 +815,7 @@ func (hc *HealthCheckImpl) GetHealthyTabletStats(target *querypb.Target) []*Tabl // The returned array is owned by the caller. // For TabletType_MASTER, this will only return at most one entry, // the most recent tablet of type master. -func (hc *HealthCheckImpl) getTabletStats(target *querypb.Target) []*TabletHealth { +func (hc *HealthCheckImpl) getTabletStats(target *query.Target) []*TabletHealth { var result []*TabletHealth ths := hc.healthData[hc.keyFromTarget(target)] for _, th := range ths { @@ -984,8 +827,8 @@ func (hc *HealthCheckImpl) getTabletStats(target *querypb.Target) []*TabletHealt // WaitForTablets waits for at least one tablet in the given // keyspace / shard / tablet type before returning. The tablets do not // have to be healthy. It will return ctx.Err() if the context is canceled. -func (hc *HealthCheckImpl) WaitForTablets(ctx context.Context, keyspace, shard string, tabletType topodatapb.TabletType) error { - targets := []*querypb.Target{ +func (hc *HealthCheckImpl) WaitForTablets(ctx context.Context, keyspace, shard string, tabletType topodata.TabletType) error { + targets := []*query.Target{ { Keyspace: keyspace, Shard: shard, @@ -999,12 +842,12 @@ func (hc *HealthCheckImpl) WaitForTablets(ctx context.Context, keyspace, shard s // each given target before returning. // It will return ctx.Err() if the context is canceled. // It will return an error if it can't read the necessary topology records. -func (hc *HealthCheckImpl) WaitForAllServingTablets(ctx context.Context, targets []*querypb.Target) error { +func (hc *HealthCheckImpl) WaitForAllServingTablets(ctx context.Context, targets []*query.Target) error { return hc.waitForTablets(ctx, targets, true) } // waitForTablets is the internal method that polls for tablets. -func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*querypb.Target, requireServing bool) error { +func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*query.Target, requireServing bool) error { for { // We nil targets as we find them. allPresent := true @@ -1044,10 +887,10 @@ func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*queryp // Target includes cell which we ignore here // because tabletStatsCache is intended to be per-cell -func (hc *HealthCheckImpl) keyFromTarget(target *querypb.Target) string { +func (hc *HealthCheckImpl) keyFromTarget(target *query.Target) string { return fmt.Sprintf("%s.%s.%d", target.Keyspace, target.Shard, target.TabletType) } -func (hc *HealthCheckImpl) keyFromTablet(tablet *topodatapb.Tablet) string { +func (hc *HealthCheckImpl) keyFromTablet(tablet *topodata.Tablet) string { return fmt.Sprintf("%s.%s.%d", tablet.Keyspace, tablet.Shard, tablet.Type) } diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go index a7c9b68d59a..bbf2f3ab15c 100644 --- a/go/vt/discovery/healthcheck_test.go +++ b/go/vt/discovery/healthcheck_test.go @@ -21,10 +21,13 @@ import ( "flag" "fmt" "html/template" - "strings" "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/topo/memorytopo" "golang.org/x/net/context" @@ -45,18 +48,201 @@ func init() { func TestHealthCheck(t *testing.T) { ts := memorytopo.NewServer("cell") + hc := createTestHc(ts) tablet := topo.NewTablet(0, "cell", "a") + tablet.Keyspace = "k" + tablet.Shard = "s" tablet.PortMap["vt"] = 1 + tablet.Type = topodatapb.TabletType_REPLICA + tabletAlias := topoproto.TabletAliasString(tablet.Alias) input := make(chan *querypb.StreamHealthResponse) - createFakeConn(tablet, input) + conn := createFakeConn(tablet, input) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) - hc := createTestHc(ts) + // close healthcheck + defer hc.Close() + testChecksum(t, 0, hc.stateChecksum()) hc.AddTablet(tablet) t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + // testChecksum(t, 2829991735, hc.stateChecksum()) + + // Immediately after AddTablet() there will be the first notification. + want := &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: false, + Stats: nil, + MasterTermStartTime: 0, + } + startTime := time.Now() + timeout := 2 * time.Second + for { + if hc.healthByAlias[tabletAlias] != nil { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for initial health") + } + time.Sleep(10 * time.Millisecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + + shr := &querypb.StreamHealthResponse{ + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5}, + } + input <- shr + t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.5}}`) + want = &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5}, + MasterTermStartTime: 0, + } + startTime = time.Now() + lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTimestamp + for { + // Health has been updated + if hc.healthByAlias[tabletAlias].lastResponseTimestamp != lastResponseTime || time.Since(startTime) > timeout { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for health update") + } + time.Sleep(10 * time.Millisecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + targetKey := hc.keyFromTarget(want.Target) + ths := hc.healthData[targetKey] + assert.NotEmpty(t, ths, "healthData is empty") + assert.True(t, want.DeepEqual(ths[tabletAlias]), "healthData contains wrong tabletHealth") + + tcsl := hc.CacheStatus() + tcslWant := TabletsCacheStatusList{{ + Cell: "cell", + Target: want.Target, + TabletsStats: TabletStatsList{{ + Tablet: tablet, + Target: want.Target, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5}, + MasterTermStartTime: 0, + }}, + }} + assert.True(t, tcslWant.deepEqual(tcsl), "Incorrect cache status:\n Expected: %+v\n Actual: %+v", tcslWant[0], tcsl[0]) + // testChecksum(t, 3487343103, hc.stateChecksum()) + + // TabletType changed, should get both old and new event + shr = &querypb.StreamHealthResponse{ + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, + Serving: true, + TabletExternallyReparentedTimestamp: 10, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + want = &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{ + Keyspace: "k", + Shard: "s", + TabletType: topodatapb.TabletType_MASTER, + }, + Serving: true, + Conn: conn, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + MasterTermStartTime: 10, + } + input <- shr + startTime = time.Now() + lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTimestamp + for { + // Health has been updated + if hc.healthByAlias[tabletAlias].lastResponseTimestamp != lastResponseTime || time.Since(startTime) > timeout { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for health update") + } + time.Sleep(10 * time.Millisecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + // testChecksum(t, 2773351292, hc.stateChecksum()) + + err := checkErrorCounter("k", "s", topodatapb.TabletType_MASTER, 0) + require.NoError(t, err, "error checking error counter") + + // Serving & RealtimeStats changed + shr = &querypb.StreamHealthResponse{ + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: false, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3}, + } + want = &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: false, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3}, + MasterTermStartTime: 0, + } + input <- shr + t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.3}}`) + startTime = time.Now() + lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTimestamp + for { + // Health has been updated + if hc.healthByAlias[tabletAlias].lastResponseTimestamp != lastResponseTime || time.Since(startTime) > timeout { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for health update") + } + time.Sleep(10 * time.Millisecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + // testChecksum(t, 2829991735, hc.stateChecksum()) + + // HealthError + shr = &querypb.StreamHealthResponse{ + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3}, + } + want = &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: false, + Stats: &querypb.RealtimeStats{HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3}, + MasterTermStartTime: 0, + LastError: fmt.Errorf("vttablet error: some error"), + } + input <- shr + t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3}}`) + startTime = time.Now() + lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTimestamp + for { + // Health has been updated + if hc.healthByAlias[tabletAlias].lastResponseTimestamp != lastResponseTime || time.Since(startTime) > timeout { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for health update") + } + time.Sleep(10 * time.Millisecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + + testChecksum(t, 1027934207, hc.stateChecksum()) // unchanged + + // remove tablet + hc.deleteConn(tablet) + t.Logf(`hc.RemoveTablet({Host: "a", PortMap: {"vt": 1}})`) + assert.Nil(t, hc.healthByAlias[tabletAlias], "Wrong tablet health") + testChecksum(t, 0, hc.stateChecksum()) - // close healthcheck - hc.Close() } func TestHealthCheckStreamError(t *testing.T) { @@ -146,13 +332,10 @@ func TestTemplate(t *testing.T) { } templ := template.New("").Funcs(status.StatusFuncs) templ, err := templ.Parse(HealthCheckTemplate) - if err != nil { - t.Fatalf("error parsing template: %v", err) - } + require.Nil(t, err, "error parsing template") wr := &bytes.Buffer{} - if err := templ.Execute(wr, []*TabletsCacheStatus{tcs}); err != nil { - t.Fatalf("error executing template: %v", err) - } + err = templ.Execute(wr, []*TabletsCacheStatus{tcs}) + require.Nil(t, err, "error executing template") } func TestDebugURLFormatting(t *testing.T) { @@ -176,17 +359,12 @@ func TestDebugURLFormatting(t *testing.T) { } templ := template.New("").Funcs(status.StatusFuncs) templ, err := templ.Parse(HealthCheckTemplate) - if err != nil { - t.Fatalf("error parsing template: %v", err) - } + require.Nil(t, err, "error parsing template") wr := &bytes.Buffer{} - if err := templ.Execute(wr, []*TabletsCacheStatus{tcs}); err != nil { - t.Fatalf("error executing template: %v", err) - } + err = templ.Execute(wr, []*TabletsCacheStatus{tcs}) + require.Nil(t, err, "error executing template") expectedURL := `"https://host.bastion.cell.corp"` - if !strings.Contains(wr.String(), expectedURL) { - t.Fatalf("output missing formatted URL, expectedURL: %s , output: %s", expectedURL, wr.String()) - } + require.Contains(t, wr.String(), expectedURL, "output missing formatted URL") } func tabletDialer(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) { diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 66d3ff0b01f..c387165a14c 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -6,8 +6,16 @@ import ( "fmt" "strings" "sync" + "time" + "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/topotools" + "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/queryservice" + "vitess.io/vitess/go/vt/vttablet/tabletconn" "github.com/golang/protobuf/proto" "vitess.io/vitess/go/netutil" @@ -18,6 +26,7 @@ import ( // TabletHealth maintains the health status of a tablet. A map of this // structure is maintained in HealthCheckImpl. type TabletHealth struct { + ctx context.Context // cancelFunc must be called before discarding TabletHealth. // This will ensure that the associated checkConn goroutine will terminate. cancelFunc context.CancelFunc @@ -42,6 +51,9 @@ type TabletHealth struct { // LastError is the error we last saw when trying to get the // tablet's healthcheck. LastError error + // possibly delete both these + loggedServingState bool + lastResponseTimestamp time.Time // timestamp of the last healthcheck response } // String is defined because we want to print a []*TabletHealth array nicely. @@ -139,3 +151,169 @@ func (th *TabletHealth) deleteConnLocked() { func (th *TabletHealth) isHealthy() bool { return th.Serving && th.LastError == nil && th.Stats != nil && !IsReplicationLagVeryHigh(th) } + +// setServingState sets the tablet state to the given value. +// +// If the state changes, it logs the change so that failures +// from the health check connection are logged the first time, +// but don't continue to log if the connection stays down. +// +// th.mu must be locked before calling this function +func (th *TabletHealth) setServingState(serving bool, reason string) { + if !th.loggedServingState || (serving != th.Serving) { + // Emit the log from a separate goroutine to avoid holding + // the th lock while logging is happening + go log.Infof("HealthCheckUpdate(Serving State): tablet: %v serving => %v for %v/%v (%v) reason: %s", + topotools.TabletIdent(th.Tablet), + serving, + th.Tablet.GetKeyspace(), + th.Tablet.GetShard(), + th.Target.GetTabletType(), + reason, + ) + th.loggedServingState = true + } + th.Serving = serving +} + +// stream streams healthcheck responses to callback. +func (th *TabletHealth) stream(ctx context.Context, callback func(*query.StreamHealthResponse) error) error { + th.mu.Lock() + if th.Conn == nil { + conn, err := tabletconn.GetDialer()(th.Tablet, grpcclient.FailFast(true)) + if err != nil { + th.LastError = err + th.mu.Unlock() + return nil + } + th.Conn = conn + th.LastError = nil + } + conn := th.Conn + th.mu.Unlock() + + err := conn.StreamHealth(ctx, callback) + if err != nil { + th.mu.Lock() + log.Warningf("tablet %v healthcheck stream error: %v", th.Tablet.Alias, err) + th.setServingState(false, err.Error()) + th.LastError = err + th.Conn.Close(ctx) + th.Conn = nil + th.mu.Unlock() + } + return err +} + +// processResponse reads one health check response, and updates health +func (th *TabletHealth) processResponse(hc *HealthCheckImpl, shr *query.StreamHealthResponse) error { + select { + case <-th.ctx.Done(): + return th.ctx.Err() + default: + } + + // Check for invalid data, better than panicking. + if shr.Target == nil || shr.RealtimeStats == nil { + return fmt.Errorf("health stats is not valid: %v", shr) + } + + // an app-level error from tablet, force serving state. + var healthErr error + serving := shr.Serving + if shr.RealtimeStats.HealthError != "" { + healthErr = fmt.Errorf("vttablet error: %v", shr.RealtimeStats.HealthError) + serving = false + } + + if shr.TabletAlias != nil && !proto.Equal(shr.TabletAlias, th.Tablet.Alias) { + // TabletAlias change means that the host:port has been taken over by another tablet + // We could cancel / exit the healthcheck for this tablet right away + // However, we defer it until the next topo refresh informs us of the change because that is + // the only way to discover the new host/port + return vterrors.New(vtrpc.Code_FAILED_PRECONDITION, fmt.Sprintf("health stats mismatch, tablet %+v alias does not match response alias %v", th.Tablet, shr.TabletAlias)) + // TODO(deepthi): delete healthcheck + } + + th.mu.Lock() + currentTablet := th.Tablet + // check whether this is a trivial update so as to update healthy map + trivialNonMasterUpdate := th.LastError == nil && th.Serving && shr.RealtimeStats.HealthError == "" && shr.Serving && + currentTablet.Type != topodata.TabletType_MASTER && currentTablet.Type == shr.Target.TabletType + isMasterUpdate := currentTablet.Type == topodata.TabletType_MASTER && shr.Target.TabletType == topodata.TabletType_MASTER + th.mu.Unlock() + + // hc.healthByAlias is authoritative, it should be updated + hc.mu.Lock() + tabletAlias := topoproto.TabletAliasString(th.Tablet.Alias) + // this will only change the first time, but it's easiest to set it always rather than check and set + hc.healthByAlias[tabletAlias] = th + hc.mu.Unlock() + + hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) + if currentTablet.Type != shr.Target.TabletType || currentTablet.Keyspace != shr.Target.Keyspace || currentTablet.Shard != shr.Target.Shard { + // keyspace and shard are not expected to change, but just in case ... + // hc still has this TabletHealth in the wrong target (because tabletType changed) + oldTargetKey := hc.keyFromTablet(currentTablet) + newTargetKey := hc.keyFromTarget(shr.Target) + tabletAlias := topoproto.TabletAliasString(currentTablet.Alias) + hc.mu.Lock() + delete(hc.healthData[oldTargetKey], tabletAlias) + _, ok := hc.healthData[newTargetKey] + if !ok { + hc.healthData[newTargetKey] = make(map[string]*TabletHealth) + } + hc.healthData[newTargetKey][tabletAlias] = th + hc.mu.Unlock() + } + + // Update our record + th.lastResponseTimestamp = time.Now() + th.mu.Lock() + defer th.mu.Unlock() + th.Target = shr.Target + th.MasterTermStartTime = shr.TabletExternallyReparentedTimestamp + th.Stats = shr.RealtimeStats + th.LastError = healthErr + reason := "healthCheck update" + if healthErr != nil { + reason = "healthCheck update error: " + healthErr.Error() + } + th.setServingState(serving, reason) + + targetKey := hc.keyFromTarget(shr.Target) + if !trivialNonMasterUpdate { + all := hc.healthData[targetKey] + allArray := make([]*TabletHealth, 0, len(all)) + for _, s := range all { + allArray = append(allArray, s) + } + hc.healthy[targetKey] = FilterStatsByReplicationLag(allArray) + } + if isMasterUpdate { + if len(hc.healthy[targetKey]) == 0 { + hc.healthy[targetKey] = append(hc.healthy[targetKey], th) + } else { + // We already have one up server, see if we + // need to replace it. + if th.MasterTermStartTime < hc.healthy[targetKey][0].MasterTermStartTime { + log.Warningf("not marking healthy master %s as Up for %s because its MasterTermStartTime is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", + topoproto.TabletAliasString(currentTablet.Alias), + topoproto.KeyspaceShardString(currentTablet.Keyspace, currentTablet.Shard), + topoproto.TabletAliasString(hc.healthy[targetKey][0].Tablet.Alias), + th.MasterTermStartTime, + hc.healthy[targetKey][0].MasterTermStartTime) + } else { + // Just replace it. + hc.healthy[targetKey][0] = th + } + } + } + // and notify downstream for master change + if shr.Target.TabletType == topodata.TabletType_MASTER { + if hc.masterCallback != nil { + hc.masterCallback(th) + } + } + return nil +} diff --git a/go/vt/discovery/tablets_cache_status.go b/go/vt/discovery/tablets_cache_status.go index 16335f679df..7261bb9708e 100644 --- a/go/vt/discovery/tablets_cache_status.go +++ b/go/vt/discovery/tablets_cache_status.go @@ -6,6 +6,8 @@ import ( "sort" "strings" + "github.com/gogo/protobuf/proto" + querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" @@ -38,6 +40,19 @@ func (tsl TabletStatsList) Swap(i, j int) { tsl[i], tsl[j] = tsl[j], tsl[i] } +func (tsl TabletStatsList) deepEqual(other TabletStatsList) bool { + if len(tsl) != len(other) { + return false + } + for i, th := range tsl { + o := other[i] + if !th.DeepEqual(o) { + return false + } + } + return true +} + // StatusAsHTML returns an HTML version of the status. func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML { tLinks := make([]string, 0, 1) @@ -64,6 +79,12 @@ func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML { return template.HTML(strings.Join(tLinks, "
")) } +func (tcs *TabletsCacheStatus) deepEqual(otcs *TabletsCacheStatus) bool { + return tcs.Cell == otcs.Cell && + proto.Equal(tcs.Target, otcs.Target) && + tcs.TabletsStats.deepEqual(otcs.TabletsStats) +} + // TabletsCacheStatusList is used for sorting. type TabletsCacheStatusList []*TabletsCacheStatus @@ -82,3 +103,16 @@ func (tcsl TabletsCacheStatusList) Less(i, j int) bool { func (tcsl TabletsCacheStatusList) Swap(i, j int) { tcsl[i], tcsl[j] = tcsl[j], tcsl[i] } + +func (tcsl TabletsCacheStatusList) deepEqual(other TabletsCacheStatusList) bool { + if len(tcsl) != len(other) { + return false + } + for i, tcs := range tcsl { + otcs := other[i] + if !tcs.deepEqual(otcs) { + return false + } + } + return true +} From fe9bdd469c126b0cb0c6e3544f5a8f65015c470e Mon Sep 17 00:00:00 2001 From: deepthi Date: Wed, 6 May 2020 12:51:18 -0700 Subject: [PATCH 18/39] healthcheck: rename TabletHealth->tabletHealthCheck, create a simpler TabletHealth struct that is returned to gateway. Unit tests Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 54 ++- go/vt/discovery/healthcheck_test.go | 365 ++++++++++++++++-- .../legacy_healthcheck_flaky_test.go | 1 + go/vt/discovery/replicationlag.go | 26 +- go/vt/discovery/replicationlag_test.go | 42 +- go/vt/discovery/tablet_health.go | 120 +++--- go/vt/vtgate/tabletgateway.go | 2 - 7 files changed, 468 insertions(+), 142 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 1972805372c..af1a78b1afb 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -173,11 +173,12 @@ type HealthCheck interface { } // HealthCheckImpl performs health checking and stores the results. -// It contains a map of TabletHealth objects per Target. -// Each TabletHealth object stores the health information for one tablet. -// A checkConn goroutine is spawned for each TabletHealth, which is responsible for -// keeping that TabletHealth up-to-date. -// If checkConn terminates for any reason, it updates TabletHealth.Up as false. If a TabletHealth +// It contains a map of tabletHealthCheck objects by Alias. +// Each tabletHealthCheck object stores the health information for one tablet. +// A checkConn goroutine is spawned for each tabletHealthCheck, which is responsible for +// keeping that tabletHealthCheck up-to-date. +// If checkConn terminates for any reason, then the corresponding tabletHealthCheck object +// is removed from the map. When a tabletHealthCheck // gets removed from the map, its cancelFunc gets called, which ensures that the associated // checkConn goroutine eventually terminates. type HealthCheckImpl struct { @@ -189,14 +190,14 @@ type HealthCheckImpl struct { // mu protects all the following fields. mu sync.Mutex // authoritative map of tabletHealth by alias - healthByAlias map[string]*TabletHealth + healthByAlias map[string]*tabletHealthCheck // a map keyed by keyspace.shard.tabletType - // contains a map of TabletHealth keyed by tablet alias for each tablet relevant to the keyspace.shard.tabletType + // contains a map of tabletHealthCheck keyed by tablet alias for each tablet relevant to the keyspace.shard.tabletType // has to be kept in sync with healthByAlias - healthData map[string]map[string]*TabletHealth - // another map keyed by keyspace.shard.tabletType, this one containing a sorted list of TabletHealth - // TODO(deepthi): replace with SimpleTabletHealth - healthy map[string][]*TabletHealth + healthData map[string]map[string]*tabletHealthCheck + // another map keyed by keyspace.shard.tabletType, this one containing a sorted list of tabletHealthCheck + // TODO(deepthi): replace with TabletHealth + healthy map[string][]*tabletHealthCheck // connsWG keeps track of all launched Go routines that monitor tablet connections. connsWG sync.WaitGroup // topology watchers that inform healthcheck of tablets being added and deleted @@ -208,11 +209,6 @@ type HealthCheckImpl struct { cellAliases map[string]string } -//type SimpleTabletHealth struct { -// TabletAlias string -// Conn queryservice.QueryService -//} - // NewHealthCheck creates a new HealthCheck object. // Parameters: // retryDelay. @@ -226,6 +222,8 @@ type HealthCheckImpl struct { // The topology server that this healthcheck object can use to retrieve cell or tablet information // localCell. // The localCell for this healthcheck +// callback. +// A function to call when there is a master change. Used to notify vtgate's buffer to stop buffering. func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string, callback func(health *TabletHealth)) HealthCheck { log.Infof("loading tablets for cells: %v", *CellsToWatch) @@ -235,9 +233,9 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur retryDelay: retryDelay, healthCheckTimeout: healthCheckTimeout, masterCallback: callback, - healthByAlias: make(map[string]*TabletHealth), - healthData: make(map[string]map[string]*TabletHealth), - healthy: make(map[string][]*TabletHealth), + healthByAlias: make(map[string]*tabletHealthCheck), + healthData: make(map[string]map[string]*tabletHealthCheck), + healthy: make(map[string][]*tabletHealthCheck), } var topoWatchers []*TopologyWatcher var filter TabletFilter @@ -520,7 +518,7 @@ func (hc *HealthCheckImpl) stateChecksum() int64 { // finalizeConn closes the health checking connection and sends the final // notification about the tablet to downstream. To be called only on exit from // checkConn(). -func (hc *HealthCheckImpl) finalizeConn(th *TabletHealth) { +func (hc *HealthCheckImpl) finalizeConn(th *tabletHealthCheck) { th.mu.Lock() defer th.mu.Unlock() th.setServingState(false, "finalizeConn closing connection") @@ -538,7 +536,7 @@ func (hc *HealthCheckImpl) finalizeConn(th *TabletHealth) { } // checkConn performs health checking on the given tablet. -func (hc *HealthCheckImpl) checkConn(th *TabletHealth) { +func (hc *HealthCheckImpl) checkConn(th *tabletHealthCheck) { defer hc.connsWG.Done() defer hc.finalizeConn(th) @@ -659,7 +657,7 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodata.Tablet) { Shard: tablet.Shard, TabletType: tablet.Type, } - th := &TabletHealth{ + th := &tabletHealthCheck{ ctx: ctx, cancelFunc: cancelFunc, Tablet: tablet, @@ -672,11 +670,12 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodata.Tablet) { // TODO: can this ever already exist? if _, ok := hc.healthByAlias[tabletAlias]; ok { log.Errorf("Program bug") + hc.mu.Unlock() return } hc.healthByAlias[tabletAlias] = th if ths, ok := hc.healthData[key]; !ok { - hc.healthData[key] = make(map[string]*TabletHealth) + hc.healthData[key] = make(map[string]*tabletHealthCheck) hc.healthData[key][tabletAlias] = th } else { // just overwrite it if it exists already? @@ -727,7 +726,7 @@ func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { } tcsMap[key] = tcs } - tcs.TabletsStats = append(tcs.TabletsStats, th) + tcs.TabletsStats = append(tcs.TabletsStats, th.SimpleCopy()) th.mu.Unlock() } return tcsMap @@ -798,12 +797,11 @@ func (hc *HealthCheckImpl) GetHealthyTabletStats(target *query.Target) []*Tablet } for _, th := range hc.healthByAlias { if th.Tablet.Type == topodata.TabletType_MASTER { - // TODO(deepthi): return SimpleHealth here - result = append(result, th.Copy()) + result = append(result, th.SimpleCopy()) return result } if th.isHealthy() { - result = append(result, th.Copy()) + result = append(result, th.SimpleCopy()) } } // healthy list needs to be sorted using replication lag algorithm @@ -819,7 +817,7 @@ func (hc *HealthCheckImpl) getTabletStats(target *query.Target) []*TabletHealth var result []*TabletHealth ths := hc.healthData[hc.keyFromTarget(target)] for _, th := range ths { - result = append(result, th.Copy()) + result = append(result, th.SimpleCopy()) } return result } diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go index bbf2f3ab15c..0fe70c1f908 100644 --- a/go/vt/discovery/healthcheck_test.go +++ b/go/vt/discovery/healthcheck_test.go @@ -21,14 +21,14 @@ import ( "flag" "fmt" "html/template" + "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" "vitess.io/vitess/go/vt/grpcclient" @@ -46,7 +46,7 @@ func init() { flag.Set("tablet_protocol", "fake_gateway") } -func TestHealthCheck(t *testing.T) { +func TestBasicHealthCheck(t *testing.T) { ts := memorytopo.NewServer("cell") hc := createTestHc(ts) tablet := topo.NewTablet(0, "cell", "a") @@ -83,9 +83,9 @@ func TestHealthCheck(t *testing.T) { if time.Since(startTime) > timeout { t.Fatal("Timed out waiting for initial health") } - time.Sleep(10 * time.Millisecond) + time.Sleep(10 * time.Microsecond) } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) shr := &querypb.StreamHealthResponse{ Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -103,22 +103,22 @@ func TestHealthCheck(t *testing.T) { MasterTermStartTime: 0, } startTime = time.Now() - lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTimestamp + lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTime() for { // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTimestamp != lastResponseTime || time.Since(startTime) > timeout { + if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { break } if time.Since(startTime) > timeout { t.Fatal("Timed out waiting for health update") } - time.Sleep(10 * time.Millisecond) + time.Sleep(10 * time.Microsecond) } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) targetKey := hc.keyFromTarget(want.Target) ths := hc.healthData[targetKey] assert.NotEmpty(t, ths, "healthData is empty") - assert.True(t, want.DeepEqual(ths[tabletAlias]), "healthData contains wrong tabletHealth") + assert.True(t, want.DeepEqual(ths[tabletAlias].SimpleCopy()), "healthData contains wrong tabletHealth") tcsl := hc.CacheStatus() tcslWant := TabletsCacheStatusList{{ @@ -156,18 +156,18 @@ func TestHealthCheck(t *testing.T) { } input <- shr startTime = time.Now() - lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTimestamp + lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTime() for { // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTimestamp != lastResponseTime || time.Since(startTime) > timeout { + if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { break } if time.Since(startTime) > timeout { t.Fatal("Timed out waiting for health update") } - time.Sleep(10 * time.Millisecond) + time.Sleep(10 * time.Microsecond) } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) // testChecksum(t, 2773351292, hc.stateChecksum()) err := checkErrorCounter("k", "s", topodatapb.TabletType_MASTER, 0) @@ -190,18 +190,18 @@ func TestHealthCheck(t *testing.T) { input <- shr t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.3}}`) startTime = time.Now() - lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTimestamp + lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTime() for { // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTimestamp != lastResponseTime || time.Since(startTime) > timeout { + if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { break } if time.Since(startTime) > timeout { t.Fatal("Timed out waiting for health update") } - time.Sleep(10 * time.Millisecond) + time.Sleep(10 * time.Microsecond) } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) // testChecksum(t, 2829991735, hc.stateChecksum()) // HealthError @@ -222,18 +222,18 @@ func TestHealthCheck(t *testing.T) { input <- shr t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3}}`) startTime = time.Now() - lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTimestamp + lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTime() for { // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTimestamp != lastResponseTime || time.Since(startTime) > timeout { + if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { break } if time.Since(startTime) > timeout { t.Fatal("Timed out waiting for health update") } - time.Sleep(10 * time.Millisecond) + time.Sleep(10 * time.Microsecond) } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) testChecksum(t, 1027934207, hc.stateChecksum()) // unchanged @@ -247,16 +247,91 @@ func TestHealthCheck(t *testing.T) { func TestHealthCheckStreamError(t *testing.T) { ts := memorytopo.NewServer("cell") + hc := createTestHc(ts) tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 + tabletAlias := topoproto.TabletAliasString(tablet.Alias) input := make(chan *querypb.StreamHealthResponse) fc := createFakeConn(tablet, input) fc.errCh = make(chan error) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) - hc := createTestHc(ts) hc.AddTablet(tablet) t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + // Immediately after AddTablet() there will be the first notification. + want := &tabletHealthCheck{ + Tablet: tablet, + Target: &querypb.Target{}, + Serving: false, + MasterTermStartTime: 0, + } + startTime := time.Now() + timeout := 2 * time.Second + for { + if hc.healthByAlias[tabletAlias] != nil { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for initial health") + } + time.Sleep(10 * time.Microsecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + + // one tablet after receiving a StreamHealthResponse + shr := &querypb.StreamHealthResponse{ + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + want = &tabletHealthCheck{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + MasterTermStartTime: 0, + } + input <- shr + t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) + startTime = time.Now() + lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTime() + for { + // Health has been updated + if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for health update") + } + time.Sleep(10 * time.Microsecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + + // Stream error + fc.errCh <- fmt.Errorf("some stream error") + want = &tabletHealthCheck{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: false, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + MasterTermStartTime: 0, + LastError: fmt.Errorf("some stream error"), + } + startTime = time.Now() + lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTime() + for { + // Health has been updated + if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for health update") + } + time.Sleep(10 * time.Microsecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + // close healthcheck hc.Close() } @@ -266,15 +341,75 @@ func TestHealthCheckVerifiesTabletAlias(t *testing.T) { t.Logf("starting") tablet := topo.NewTablet(1, "cell", "a") tablet.PortMap["vt"] = 1 + tabletAlias := topoproto.TabletAliasString(tablet.Alias) input := make(chan *querypb.StreamHealthResponse, 1) - createFakeConn(tablet, input) - + fc := createFakeConn(tablet, input) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) - hc := createTestHc(ts) hc.AddTablet(tablet) t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + // Immediately after AddTablet() there will be the first notification. + want := &tabletHealthCheck{ + Tablet: tablet, + Target: &querypb.Target{}, + Serving: false, + MasterTermStartTime: 0, + } + startTime := time.Now() + timeout := 2 * time.Second + for { + if hc.healthByAlias[tabletAlias] != nil { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for initial health") + } + time.Sleep(10 * time.Microsecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + + input <- &querypb.StreamHealthResponse{ + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, + TabletAlias: &topodatapb.TabletAlias{Uid: 20, Cell: "cellb"}, + Serving: true, + TabletExternallyReparentedTimestamp: 10, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + + ticker := time.NewTicker(2 * time.Second) + select { + case err := <-fc.cbErrCh: + t.Logf("<-fc.cbErrCh: %v", err) + if prefix := "health stats mismatch"; !strings.HasPrefix(err.Error(), prefix) { + t.Fatalf("wrong error, got %v; want prefix %v", err, prefix) + } + case <-ticker.C: + t.Fatalf("Timed out waiting for StreamHealth to return a health stats mismatch error") + } + + input <- &querypb.StreamHealthResponse{ + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, + TabletAlias: &topodatapb.TabletAlias{Uid: 1, Cell: "cell"}, + Serving: true, + TabletExternallyReparentedTimestamp: 10, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + + startTime = time.Now() + lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTime() + for { + // Health has been updated + if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for health update") + } + time.Sleep(10 * time.Microsecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + // close healthcheck hc.Close() } @@ -283,33 +418,199 @@ func TestHealthCheckVerifiesTabletAlias(t *testing.T) { // routines to finish and the listener won't be called anymore. func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { ts := memorytopo.NewServer("cell") + hc := createTestHc(ts) tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 + tabletAlias := topoproto.TabletAliasString(tablet.Alias) input := make(chan *querypb.StreamHealthResponse, 1) createFakeConn(tablet, input) - t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) - - hc := createTestHc(ts) hc.AddTablet(tablet) t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + // Immediately after AddTablet() there will be the first notification. + want := &tabletHealthCheck{ + Tablet: tablet, + Target: &querypb.Target{}, + Serving: false, + MasterTermStartTime: 0, + } + startTime := time.Now() + timeout := 2 * time.Second + for { + if hc.healthByAlias[tabletAlias] != nil { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for initial health") + } + time.Sleep(10 * time.Microsecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + + // one tablet after receiving a StreamHealthResponse + shr := &querypb.StreamHealthResponse{ + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + want = &tabletHealthCheck{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + MasterTermStartTime: 0, + } + input <- shr + t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) + startTime = time.Now() + lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTime() + for { + // Health has been updated + if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for health update") + } + time.Sleep(10 * time.Microsecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + + // Change input to distinguish between stats sent before and after Close(). + shr.TabletExternallyReparentedTimestamp = 11 + // Close the healthcheck. Tablet connections are closed asynchronously and + // Close() will block until all Go routines (one per connection) are done. hc.Close() + // Try to send more updates. They should be ignored and nothing should change + // Note that this code is racy by nature. If there is a regression, it should + // fail in some cases. + input <- shr + t.Logf(`input <- %v`, shr) + assert.Nil(t, hc.healthByAlias, "health data should be nil") } func TestHealthCheckTimeout(t *testing.T) { ts := memorytopo.NewServer("cell") + hc := createTestHc(ts) timeout := 500 * time.Millisecond + hc.healthCheckTimeout = 2 * timeout tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 + tabletAlias := topoproto.TabletAliasString(tablet.Alias) input := make(chan *querypb.StreamHealthResponse) - createFakeConn(tablet, input) + fc := createFakeConn(tablet, input) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) - hc := createTestHc(ts) - hc.retryDelay = timeout hc.AddTablet(tablet) t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + // Immediately after AddTablet() there will be the first notification. + want := &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{}, + Serving: false, + MasterTermStartTime: 0, + } + startTime := time.Now() + for { + if hc.healthByAlias[tabletAlias] != nil { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for initial health") + } + time.Sleep(10 * time.Microsecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + // one tablet after receiving a StreamHealthResponse + shr := &querypb.StreamHealthResponse{ + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + want = &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + MasterTermStartTime: 0, + } + input <- shr + t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) + startTime = time.Now() + lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTime() + for { + // Health has been updated + if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for health update") + } + time.Sleep(10 * time.Microsecond) + } + assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + + if err := checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 0); err != nil { + t.Errorf("%v", err) + } + + // wait for timeout period + time.Sleep(hc.healthCheckTimeout + 100*time.Millisecond) + t.Logf(`Sleep(1.1 * timeout)`) + res := hc.healthByAlias[tabletAlias] + if res.Serving { + t.Errorf(`tabletHealthCheck: %+v; want not serving`, res) + } + + if err := checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 1); err != nil { + t.Errorf("%v", err) + } + + if !fc.isCanceled() { + t.Errorf("StreamHealth should be canceled after timeout, but is not") + } + + // repeat the wait. It will timeout one more time trying to get the connection. + fc.resetCanceledFlag() + time.Sleep(hc.healthCheckTimeout) + t.Logf(`Sleep(timeout)`) + + res = hc.healthByAlias[tabletAlias] + if res.Serving { + t.Errorf(`tabletHealthCheck: %+v; want not serving`, res) + } + + if err := checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 2); err != nil { + t.Errorf("%v", err) + } + + if !fc.isCanceled() { + t.Errorf("StreamHealth should be canceled again after timeout") + } + + // send a healthcheck response, it should be serving again + fc.resetCanceledFlag() + input <- shr + t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) + + // wait for the exponential backoff to wear off and health monitoring to resume. + startTime = time.Now() + lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTime() + for { + // Health has been updated + if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { + break + } + if time.Since(startTime) > timeout { + t.Fatal("Timed out waiting for health update") + } + time.Sleep(10 * time.Microsecond) + } + res = hc.healthByAlias[tabletAlias] + assert.True(t, want.DeepEqual(res.SimpleCopy()), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) // close healthcheck hc.Close() } diff --git a/go/vt/discovery/legacy_healthcheck_flaky_test.go b/go/vt/discovery/legacy_healthcheck_flaky_test.go index 60f8a1f7c10..51ed08e7cab 100644 --- a/go/vt/discovery/legacy_healthcheck_flaky_test.go +++ b/go/vt/discovery/legacy_healthcheck_flaky_test.go @@ -57,6 +57,7 @@ func testChecksum(t *testing.T, want, got int64) { } func TestLegacyHealthCheck(t *testing.T) { + hcErrorCounters.ResetAll() tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 input := make(chan *querypb.StreamHealthResponse) diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index b448d8ece7d..28f55fa6f90 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -32,18 +32,18 @@ var ( // IsReplicationLagHigh verifies that the given LegacytabletHealth refers to a tablet with high // replication lag, i.e. higher than the configured discovery_low_replication_lag flag. -func IsReplicationLagHigh(tabletHealth *TabletHealth) bool { +func IsReplicationLagHigh(tabletHealth *tabletHealthCheck) bool { return float64(tabletHealth.Stats.SecondsBehindMaster) > lowReplicationLag.Seconds() } // IsReplicationLagVeryHigh verifies that the given LegacytabletHealth refers to a tablet with very high // replication lag, i.e. higher than the configured discovery_high_replication_lag_minimum_serving flag. -func IsReplicationLagVeryHigh(tabletHealth *TabletHealth) bool { +func IsReplicationLagVeryHigh(tabletHealth *tabletHealthCheck) bool { return float64(tabletHealth.Stats.SecondsBehindMaster) > highReplicationLagMinServing.Seconds() } -// FilterStatsByReplicationLag filters the list of TabletHealth by TabletHealth.Stats.SecondsBehindMaster. -// Note that TabletHealth that is non-serving or has error is ignored. +// FilterStatsByReplicationLag filters the list of tabletHealthCheck by tabletHealthCheck.Stats.SecondsBehindMaster. +// Note that tabletHealthCheck that is non-serving or has error is ignored. // // The simplified logic: // - Return tablets that have lag <= lowReplicationLag. @@ -67,7 +67,7 @@ func IsReplicationLagVeryHigh(tabletHealth *TabletHealth) bool { // The default for this is 2h, same as the discovery_high_replication_lag_minimum_serving here. // * degraded_threshold: this is only used by vttablet for display. It should match // discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. -func FilterStatsByReplicationLag(tabletHealthList []*TabletHealth) []*TabletHealth { +func FilterStatsByReplicationLag(tabletHealthList []*tabletHealthCheck) []*tabletHealthCheck { if !*legacyReplicationLagAlgorithm { return filterStatsByLag(tabletHealthList) } @@ -81,7 +81,7 @@ func FilterStatsByReplicationLag(tabletHealthList []*TabletHealth) []*TabletHeal } -func filterStatsByLag(tabletHealthList []*TabletHealth) []*TabletHealth { +func filterStatsByLag(tabletHealthList []*tabletHealthCheck) []*tabletHealthCheck { list := make([]tabletLagSnapshot, 0, len(tabletHealthList)) // filter non-serving tablets and those with very high replication lag for _, ts := range tabletHealthList { @@ -98,7 +98,7 @@ func filterStatsByLag(tabletHealthList []*TabletHealth) []*TabletHealth { sort.Sort(tabletLagSnapshotList(list)) // Pick those with low replication lag, but at least minNumTablets tablets regardless. - res := make([]*TabletHealth, 0, len(list)) + res := make([]*tabletHealthCheck, 0, len(list)) for i := 0; i < len(list); i++ { if !IsReplicationLagHigh(list[i].ts) || i < *minNumTablets { res = append(res, list[i].ts) @@ -107,8 +107,8 @@ func filterStatsByLag(tabletHealthList []*TabletHealth) []*TabletHealth { return res } -func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*TabletHealth) []*TabletHealth { - list := make([]*TabletHealth, 0, len(tabletHealthList)) +func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*tabletHealthCheck) []*tabletHealthCheck { + list := make([]*tabletHealthCheck, 0, len(tabletHealthList)) // filter non-serving tablets for _, ts := range tabletHealthList { if !ts.Serving || ts.LastError != nil || ts.Stats == nil { @@ -132,7 +132,7 @@ func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*TabletHealth) []*Ta } // filter those affecting "mean" lag significantly // calculate mean for all tablets - res := make([]*TabletHealth, 0, len(list)) + res := make([]*tabletHealthCheck, 0, len(list)) m, _ := mean(list, -1) for i, ts := range list { // calculate mean by excluding ith tablet @@ -174,7 +174,7 @@ func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*TabletHealth) []*Ta sort.Sort(byReplag(snapshots)) // Pick the first minNumTablets tablets. - res = make([]*TabletHealth, 0, *minNumTablets) + res = make([]*tabletHealthCheck, 0, *minNumTablets) for i := 0; i < min(*minNumTablets, len(snapshots)); i++ { res = append(res, snapshots[i].ts) } @@ -188,7 +188,7 @@ func (a byReplag) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byReplag) Less(i, j int) bool { return a[i].replag < a[j].replag } type tabletLagSnapshot struct { - ts *TabletHealth + ts *tabletHealthCheck replag uint32 } type tabletLagSnapshotList []tabletLagSnapshot @@ -206,7 +206,7 @@ func min(a, b int) int { // mean calculates the mean value over the given list, // while excluding the item with the specified index. -func mean(tabletHealthList []*TabletHealth, idxExclude int) (uint64, error) { +func mean(tabletHealthList []*tabletHealthCheck, idxExclude int) (uint64, error) { var sum uint64 var count uint64 for i, ts := range tabletHealthList { diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index c920da0cd07..83e61e77402 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -31,17 +31,17 @@ func testSetMinNumTablets(newMin int) { func TestFilterByReplicationLagUnhealthy(t *testing.T) { // 1 healthy serving tablet, 1 not healhty - ts1 := &TabletHealth{ + ts1 := &tabletHealthCheck{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{}, } - ts2 := &TabletHealth{ + ts2 := &tabletHealthCheck{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: false, Stats: &querypb.RealtimeStats{}, } - got := FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2}) + got := FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2}) if len(got) != 1 { t.Errorf("len(FilterStatsByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}])) = %v, want 1", len(got)) } @@ -102,9 +102,9 @@ func TestFilterByReplicationLag(t *testing.T) { } for _, tc := range cases { - lts := make([]*TabletHealth, len(tc.input)) + lts := make([]*tabletHealthCheck, len(tc.input)) for i, lag := range tc.input { - lts[i] = &TabletHealth{ + lts[i] = &tabletHealthCheck{ Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, @@ -130,52 +130,52 @@ func TestFilterByReplicationLagThreeTabletMin(t *testing.T) { // Use at least 3 tablets if possible testSetMinNumTablets(3) // lags of (1s, 1s, 10m, 11m) - returns at least32 items where the slightly delayed ones that are returned are the 10m and 11m ones. - ts1 := &TabletHealth{ + ts1 := &tabletHealthCheck{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &TabletHealth{ + ts2 := &tabletHealthCheck{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts3 := &TabletHealth{ + ts3 := &tabletHealthCheck{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts4 := &TabletHealth{ + ts4 := &tabletHealthCheck{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - got := FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2, ts3, ts4}) + got := FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts1) || !got[1].DeepEqual(ts2) || !got[2].DeepEqual(ts3) { t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } // lags of (11m, 10m, 1s, 1s) - reordered tablets returns the same 3 items where the slightly delayed one that is returned is the 10m and 11m ones. - ts1 = &TabletHealth{ + ts1 = &tabletHealthCheck{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - ts2 = &TabletHealth{ + ts2 = &tabletHealthCheck{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts3 = &TabletHealth{ + ts3 = &tabletHealthCheck{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts4 = &TabletHealth{ + ts4 = &tabletHealthCheck{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - got = FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2, ts3, ts4}) + got = FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2, ts3, ts4}) if len(got) != 3 || !got[0].DeepEqual(ts3) || !got[1].DeepEqual(ts4) || !got[2].DeepEqual(ts2) { t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) } @@ -187,32 +187,32 @@ func TestFilterStatsByReplicationLagOneTabletMin(t *testing.T) { // Use at least 1 tablets if possible testSetMinNumTablets(1) // lags of (1s, 100m) - return only healthy tablet if that is all that is available. - ts1 := &TabletHealth{ + ts1 := &tabletHealthCheck{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &TabletHealth{ + ts2 := &tabletHealthCheck{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got := FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2}) + got := FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { t.Errorf("FilterStatsByReplicationLag([1s, 100m]) = %+v, want [1s]", got) } // lags of (1m, 100m) - return only healthy tablet if that is all that is healthy enough. - ts1 = &TabletHealth{ + ts1 = &tabletHealthCheck{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60}, } - ts2 = &TabletHealth{ + ts2 = &tabletHealthCheck{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got = FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2}) + got = FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2}) if len(got) != 1 || !got[0].DeepEqual(ts1) { t.Errorf("FilterStatsByReplicationLag([1m, 100m]) = %+v, want [1m]", got) } diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index c387165a14c..8120e01d695 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -23,11 +23,11 @@ import ( "vitess.io/vitess/go/vt/proto/topodata" ) -// TabletHealth maintains the health status of a tablet. A map of this +// tabletHealthCheck maintains the health status of a tablet. A map of this // structure is maintained in HealthCheckImpl. -type TabletHealth struct { +type tabletHealthCheck struct { ctx context.Context - // cancelFunc must be called before discarding TabletHealth. + // cancelFunc must be called before discarding tabletHealthCheck. // This will ensure that the associated checkConn goroutine will terminate. cancelFunc context.CancelFunc // Tablet is the tablet object that was sent to HealthCheck.AddTablet. @@ -56,33 +56,16 @@ type TabletHealth struct { lastResponseTimestamp time.Time // timestamp of the last healthcheck response } -// String is defined because we want to print a []*TabletHealth array nicely. -func (th *TabletHealth) String() string { - th.mu.Lock() - defer th.mu.Unlock() - return fmt.Sprintf("TabletHealth{Tablet: %v,Target: %v,Serving: %v, MasterTermStartTime: %v, Stats: %v, LastError: %v", - th.Tablet, th.Target, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) -} - -// Copy returns a copy of TabletHealth. Note that this is not really a deep copy -// because we point to the same underlying RealtimeStats. -// That is fine because the RealtimeStats object is never changed after creation. -func (th *TabletHealth) Copy() *TabletHealth { - th.mu.Lock() - defer th.mu.Unlock() - // we have to explicitly create a new object rather than relying on assignment to make a copy for us - // the following doesn't work for synchronized objects - // t := *th - // return &t - return &TabletHealth{ - Conn: th.Conn, - Tablet: th.Tablet, - Target: th.Target, - Serving: th.Serving, - MasterTermStartTime: th.MasterTermStartTime, - Stats: th.Stats, - LastError: th.LastError, - } +// TabletHealth represents simple tablet health data that is returned to users of healthcheck. +// No synchronization is required because we always return a copy. +type TabletHealth struct { + Conn queryservice.QueryService + Tablet *topodata.Tablet + Target *query.Target + Stats *query.RealtimeStats + MasterTermStartTime int64 + LastError error + Serving bool } // DeepEqual compares two TabletHealth. Since we include protos, we @@ -99,19 +82,15 @@ func (th *TabletHealth) DeepEqual(other *TabletHealth) bool { // GetTabletHostPort formats a tablet host port address. func (th *TabletHealth) GetTabletHostPort() string { - th.mu.Lock() hostname := th.Tablet.Hostname vtPort := th.Tablet.PortMap["vt"] - th.mu.Unlock() return netutil.JoinHostPort(hostname, vtPort) } // GetHostNameLevel returns the specified hostname level. If the level does not exist it will pick the closest level. // This seems unused but can be utilized by certain url formatting templates. See getTabletDebugURL for more details. func (th *TabletHealth) GetHostNameLevel(level int) string { - th.mu.Lock() hostname := th.Tablet.Hostname - th.mu.Unlock() chunkedHostname := strings.Split(hostname, ".") if level < 0 { @@ -126,8 +105,8 @@ func (th *TabletHealth) GetHostNameLevel(level int) string { // getTabletDebugURL formats a debug url to the tablet. // It uses a format string that can be passed into the app to format // the debug URL to accommodate different network setups. It applies -// the html/template string defined to a TabletHealth object. The -// format string can refer to members and functions of TabletHealth +// the html/template string defined to a tabletHealthCheck object. The +// format string can refer to members and functions of tabletHealthCheck // like a regular html/template string. // // For instance given a tablet with hostname:port of host.dc.domain:22 @@ -141,14 +120,60 @@ func (th *TabletHealth) getTabletDebugURL() string { return buffer.String() } -func (th *TabletHealth) deleteConnLocked() { +// String is defined because we want to print a []*tabletHealthCheck array nicely. +func (th *tabletHealthCheck) String() string { + th.mu.Lock() + defer th.mu.Unlock() + return fmt.Sprintf("tabletHealthCheck{Tablet: %v,Target: %v,Serving: %v, MasterTermStartTime: %v, Stats: %v, LastError: %v", + th.Tablet, th.Target, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) +} + +func (th *tabletHealthCheck) lastResponseTime() time.Time { + th.mu.Lock() + defer th.mu.Unlock() + res := th.lastResponseTimestamp + return res +} + +// SimpleCopy returns a TabletHealth with all the necessary fields copied from tabletHealthCheck. +// Note that this is not a deep copy because we point to the same underlying RealtimeStats. +// That is fine because the RealtimeStats object is never changed after creation. +func (th *tabletHealthCheck) SimpleCopy() *TabletHealth { + // we have to explicitly create a new object rather than relying on assignment to make a copy for us + // the following doesn't work for synchronized objects + // t := *th + // return &t + return &TabletHealth{ + Conn: th.Conn, + Tablet: th.Tablet, + Target: th.Target, + Stats: th.Stats, + LastError: th.LastError, + MasterTermStartTime: th.MasterTermStartTime, + Serving: th.Serving, + } +} + +// DeepEqual compares two tabletHealthCheck. Since we include protos, we +// need to use proto.Equal on these. +func (th *tabletHealthCheck) DeepEqual(other *tabletHealthCheck) bool { + return proto.Equal(th.Tablet, other.Tablet) && + proto.Equal(th.Target, other.Target) && + th.Serving == other.Serving && + th.MasterTermStartTime == other.MasterTermStartTime && + proto.Equal(th.Stats, other.Stats) && + ((th.LastError == nil && other.LastError == nil) || + (th.LastError != nil && other.LastError != nil && th.LastError.Error() == other.LastError.Error())) +} + +func (th *tabletHealthCheck) deleteConnLocked() { th.mu.Lock() th.Conn = nil th.mu.Unlock() th.cancelFunc() } -func (th *TabletHealth) isHealthy() bool { +func (th *tabletHealthCheck) isHealthy() bool { return th.Serving && th.LastError == nil && th.Stats != nil && !IsReplicationLagVeryHigh(th) } @@ -159,7 +184,7 @@ func (th *TabletHealth) isHealthy() bool { // but don't continue to log if the connection stays down. // // th.mu must be locked before calling this function -func (th *TabletHealth) setServingState(serving bool, reason string) { +func (th *tabletHealthCheck) setServingState(serving bool, reason string) { if !th.loggedServingState || (serving != th.Serving) { // Emit the log from a separate goroutine to avoid holding // the th lock while logging is happening @@ -177,7 +202,7 @@ func (th *TabletHealth) setServingState(serving bool, reason string) { } // stream streams healthcheck responses to callback. -func (th *TabletHealth) stream(ctx context.Context, callback func(*query.StreamHealthResponse) error) error { +func (th *tabletHealthCheck) stream(ctx context.Context, callback func(*query.StreamHealthResponse) error) error { th.mu.Lock() if th.Conn == nil { conn, err := tabletconn.GetDialer()(th.Tablet, grpcclient.FailFast(true)) @@ -200,13 +225,15 @@ func (th *TabletHealth) stream(ctx context.Context, callback func(*query.StreamH th.LastError = err th.Conn.Close(ctx) th.Conn = nil + // signal that healthCheck is now up-to-date + th.lastResponseTimestamp = time.Now() th.mu.Unlock() } return err } // processResponse reads one health check response, and updates health -func (th *TabletHealth) processResponse(hc *HealthCheckImpl, shr *query.StreamHealthResponse) error { +func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.StreamHealthResponse) error { select { case <-th.ctx.Done(): return th.ctx.Err() @@ -253,7 +280,7 @@ func (th *TabletHealth) processResponse(hc *HealthCheckImpl, shr *query.StreamHe hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) if currentTablet.Type != shr.Target.TabletType || currentTablet.Keyspace != shr.Target.Keyspace || currentTablet.Shard != shr.Target.Shard { // keyspace and shard are not expected to change, but just in case ... - // hc still has this TabletHealth in the wrong target (because tabletType changed) + // hc still has this tabletHealthCheck in the wrong target (because tabletType changed) oldTargetKey := hc.keyFromTablet(currentTablet) newTargetKey := hc.keyFromTarget(shr.Target) tabletAlias := topoproto.TabletAliasString(currentTablet.Alias) @@ -261,14 +288,13 @@ func (th *TabletHealth) processResponse(hc *HealthCheckImpl, shr *query.StreamHe delete(hc.healthData[oldTargetKey], tabletAlias) _, ok := hc.healthData[newTargetKey] if !ok { - hc.healthData[newTargetKey] = make(map[string]*TabletHealth) + hc.healthData[newTargetKey] = make(map[string]*tabletHealthCheck) } hc.healthData[newTargetKey][tabletAlias] = th hc.mu.Unlock() } // Update our record - th.lastResponseTimestamp = time.Now() th.mu.Lock() defer th.mu.Unlock() th.Target = shr.Target @@ -284,7 +310,7 @@ func (th *TabletHealth) processResponse(hc *HealthCheckImpl, shr *query.StreamHe targetKey := hc.keyFromTarget(shr.Target) if !trivialNonMasterUpdate { all := hc.healthData[targetKey] - allArray := make([]*TabletHealth, 0, len(all)) + allArray := make([]*tabletHealthCheck, 0, len(all)) for _, s := range all { allArray = append(allArray, s) } @@ -312,8 +338,10 @@ func (th *TabletHealth) processResponse(hc *HealthCheckImpl, shr *query.StreamHe // and notify downstream for master change if shr.Target.TabletType == topodata.TabletType_MASTER { if hc.masterCallback != nil { - hc.masterCallback(th) + hc.masterCallback(th.SimpleCopy()) } } + // signal that healthCheck is now up-to-date + th.lastResponseTimestamp = time.Now() return nil } diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index b3609f8dc63..55621244c28 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -192,8 +192,6 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, } } - // instead of returning []*TabletHealth we can return a simpler struct - // that contains just tablet ALias and connection tablets := gw.hc.GetHealthyTabletStats(target) if len(tablets) == 0 { // fail fast if there is no tablet From 741c012728088066ce14bd18e812d5e615dd2b76 Mon Sep 17 00:00:00 2001 From: deepthi Date: Wed, 6 May 2020 14:30:41 -0700 Subject: [PATCH 19/39] healthcheck: HealthCheck cache in vtgate debug/status page Signed-off-by: deepthi --- go/cmd/vtgate/status.go | 2 +- go/cmd/vtgate/vtgate.go | 1 - go/vt/servenv/status.go | 2 -- go/vt/vtgate/discoverygateway.go | 6 ++++++ go/vt/vtgate/gateway.go | 3 +++ go/vt/vtgate/tabletgateway.go | 5 +++++ 6 files changed, 15 insertions(+), 4 deletions(-) diff --git a/go/cmd/vtgate/status.go b/go/cmd/vtgate/status.go index 564f1b839df..b2548a24360 100644 --- a/go/cmd/vtgate/status.go +++ b/go/cmd/vtgate/status.go @@ -44,7 +44,7 @@ func addStatusParts(vtg *vtgate.VTGate) { }) } else { servenv.AddStatusPart("Health Check Cache", discovery.HealthCheckTemplate, func() interface{} { - return healthCheck.CacheStatus() + return vtg.Gateway().HealthCheck().CacheStatus() }) } } diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index 9de94d31c0f..1c346be4e1a 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -43,7 +43,6 @@ var ( var resilientServer *srvtopo.ResilientServer var legacyHealthCheck discovery.LegacyHealthCheck -var healthCheck discovery.HealthCheck func init() { rand.Seed(time.Now().UnixNano()) diff --git a/go/vt/servenv/status.go b/go/vt/servenv/status.go index 1d41c1acbb5..619e75f0956 100644 --- a/go/vt/servenv/status.go +++ b/go/vt/servenv/status.go @@ -223,8 +223,6 @@ func (sp *statusPage) statusHandler(w http.ResponseWriter, r *http.Request) { if err := sp.tmpl.ExecuteTemplate(w, "status", data); err != nil { if _, ok := err.(net.Error); !ok { log.Errorf("servenv: couldn't execute template: %v", err) - log.Infof("template: %v", sp.tmpl) - log.Infof("data: %v", data) } } } diff --git a/go/vt/vtgate/discoverygateway.go b/go/vt/vtgate/discoverygateway.go index 8b03db138e1..a8ca17125e5 100644 --- a/go/vt/vtgate/discoverygateway.go +++ b/go/vt/vtgate/discoverygateway.go @@ -408,3 +408,9 @@ func NewShardError(in error, target *querypb.Target, tablet *topodatapb.Tablet) } return in } + +// HealthCheck should never be called on a DiscoveryGateway +// This exists only to satisfy the interface +func (dg *DiscoveryGateway) HealthCheck() discovery.HealthCheck { + return nil +} diff --git a/go/vt/vtgate/gateway.go b/go/vt/vtgate/gateway.go index 1eb65722db9..ed7b56b1057 100644 --- a/go/vt/vtgate/gateway.go +++ b/go/vt/vtgate/gateway.go @@ -62,6 +62,9 @@ type Gateway interface { // CacheStatus returns a list of TabletCacheStatus per shard / tablet type. CacheStatus() TabletCacheStatusList + + // HealthCheck returns a reference to the healthCheck being used by this gateway + HealthCheck() discovery.HealthCheck } // Creator is the factory method which can create the actual gateway object. diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index 55621244c28..1ff6d1204b9 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -308,3 +308,8 @@ func (gw *TabletGateway) nextTablet(cell string, tablets []*discovery.TabletHeal } return -1 } + +// HealthCheck satisfies the Gateway interface +func (gw *TabletGateway) HealthCheck() discovery.HealthCheck { + return gw.hc +} From 94e597f32c6e3e922af6101baf2da4fb15fa6c8c Mon Sep 17 00:00:00 2001 From: deepthi Date: Wed, 6 May 2020 22:14:04 -0700 Subject: [PATCH 20/39] healthcheck: refactor stream into smaller funcs, use channel for unit tests Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 57 ++- go/vt/discovery/healthcheck_test.go | 415 +++++++----------- .../legacy_healthcheck_flaky_test.go | 86 ---- go/vt/discovery/tablet_health.go | 72 +-- go/vt/discovery/tablets_cache_status.go | 2 +- 5 files changed, 258 insertions(+), 374 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index af1a78b1afb..09c7ae49c3d 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -207,6 +207,37 @@ type HealthCheckImpl struct { masterCallback func(health *TabletHealth) // cellAliases is a cache of cell aliases cellAliases map[string]string + // mutex to protect subscribers + subMu sync.Mutex + // subscribers + subscribers map[chan *TabletHealth]struct{} +} + +// Subscribe adds a listener. Only used for testing right now +func (hc *HealthCheckImpl) Subscribe() chan *TabletHealth { + hc.subMu.Lock() + defer hc.subMu.Unlock() + c := make(chan *TabletHealth, 2) + hc.subscribers[c] = struct{}{} + return c +} + +// Unsubscribe removes a listener. Only used for testing right now +func (hc *HealthCheckImpl) Unsubscribe(c chan *TabletHealth) { + hc.subMu.Lock() + defer hc.subMu.Unlock() + delete(hc.subscribers, c) +} + +func (hc *HealthCheckImpl) broadcast(th *TabletHealth) { + hc.subMu.Lock() + defer hc.subMu.Unlock() + for c := range hc.subscribers { + select { + case c <- th: + default: + } + } } // NewHealthCheck creates a new HealthCheck object. @@ -236,6 +267,7 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur healthByAlias: make(map[string]*tabletHealthCheck), healthData: make(map[string]map[string]*tabletHealthCheck), healthy: make(map[string][]*tabletHealthCheck), + subscribers: make(map[chan *TabletHealth]struct{}), } var topoWatchers []*TopologyWatcher var filter TabletFilter @@ -585,20 +617,25 @@ func (hc *HealthCheckImpl) checkConn(th *tabletHealthCheck) { // streamCancel to make sure the watcher goroutine terminates. streamCancel() - if err != nil && strings.Contains(err.Error(), "health stats mismatch") { - //finalizeConn will delete all data once this loop breaks - return + if err != nil { + if strings.Contains(err.Error(), "health stats mismatch") { + hc.deleteConn(th.Tablet) + return + } + res := th.SimpleCopy() + hc.broadcast(res) } // If there was a timeout send an error. We do this after stream has returned. // This will ensure that this update prevails over any previous message that // stream could have sent. if timedout.Get() { th.mu.Lock() - // get timestamp from error and put it into LastError and remove th.lastResponseTimestamp th.LastError = fmt.Errorf("healthcheck timed out (latest %v)", th.lastResponseTimestamp) th.setServingState(false, th.LastError.Error()) hcErrorCounters.Add([]string{th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)}, 1) + res := th.simpleCopyLocked() th.mu.Unlock() + hc.broadcast(res) } // Streaming RPC failed e.g. because vttablet was restarted or took too long. @@ -626,7 +663,7 @@ func (hc *HealthCheckImpl) deleteConn(tablet *topodata.Tablet) { // delete from authoritative map th, ok := hc.healthByAlias[tabletAlias] if !ok { - log.Warningf("Something is wrong, we have no health data for tablet: %v", tabletAlias) + log.Infof("We have no health data for tablet: %v, it might have been deleted already", tabletAlias) return } th.deleteConnLocked() @@ -634,7 +671,7 @@ func (hc *HealthCheckImpl) deleteConn(tablet *topodata.Tablet) { // delete from map by keyspace.shard.tabletType ths, ok := hc.healthData[key] if !ok { - log.Warningf("Something is wrong, we have no health data for target: %v", key) + log.Warningf("We have no health data for target: %v", key) return } delete(ths, tabletAlias) @@ -682,6 +719,8 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodata.Tablet) { ths[tabletAlias] = th } + res := th.SimpleCopy() + hc.broadcast(res) hc.connsWG.Add(1) hc.mu.Unlock() go hc.checkConn(th) @@ -718,7 +757,6 @@ func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { key := fmt.Sprintf("%v.%v.%v.%v", th.Tablet.Alias.Cell, th.Target.Keyspace, th.Target.Shard, th.Target.TabletType.String()) var tcs *TabletsCacheStatus var ok bool - th.mu.Lock() if tcs, ok = tcsMap[key]; !ok { tcs = &TabletsCacheStatus{ Cell: th.Tablet.Alias.Cell, @@ -727,7 +765,6 @@ func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { tcsMap[key] = tcs } tcs.TabletsStats = append(tcs.TabletsStats, th.SimpleCopy()) - th.mu.Unlock() } return tcsMap } @@ -743,6 +780,10 @@ func (hc *HealthCheckImpl) Close() error { for _, tw := range hc.topoWatchers { tw.Stop() } + for s := range hc.subscribers { + close(s) + } + hc.subscribers = nil // Release the lock early or a pending checkHealthCheckTimeout // cannot get a read lock on it. hc.mu.Unlock() diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go index 0fe70c1f908..b595f269739 100644 --- a/go/vt/discovery/healthcheck_test.go +++ b/go/vt/discovery/healthcheck_test.go @@ -21,10 +21,14 @@ import ( "flag" "fmt" "html/template" + "io" "strings" + "sync" "testing" "time" + "vitess.io/vitess/go/vt/vttablet/queryservice/fakes" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "vitess.io/vitess/go/vt/topo/memorytopo" @@ -46,25 +50,26 @@ func init() { flag.Set("tablet_protocol", "fake_gateway") } -func TestBasicHealthCheck(t *testing.T) { +func TestHealthCheck(t *testing.T) { ts := memorytopo.NewServer("cell") hc := createTestHc(ts) + // close healthcheck + defer hc.Close() tablet := topo.NewTablet(0, "cell", "a") tablet.Keyspace = "k" tablet.Shard = "s" tablet.PortMap["vt"] = 1 tablet.Type = topodatapb.TabletType_REPLICA - tabletAlias := topoproto.TabletAliasString(tablet.Alias) input := make(chan *querypb.StreamHealthResponse) conn := createFakeConn(tablet, input) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) - // close healthcheck - defer hc.Close() + // create a channel and subscribe to healthcheck + resultChan := hc.Subscribe() testChecksum(t, 0, hc.stateChecksum()) hc.AddTablet(tablet) t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) - // testChecksum(t, 2829991735, hc.stateChecksum()) + testChecksum(t, 1027934207, hc.stateChecksum()) // Immediately after AddTablet() there will be the first notification. want := &TabletHealth{ @@ -74,18 +79,8 @@ func TestBasicHealthCheck(t *testing.T) { Stats: nil, MasterTermStartTime: 0, } - startTime := time.Now() - timeout := 2 * time.Second - for { - if hc.healthByAlias[tabletAlias] != nil { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for initial health") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + result := <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) shr := &querypb.StreamHealthResponse{ Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -95,6 +90,7 @@ func TestBasicHealthCheck(t *testing.T) { } input <- shr t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.5}}`) + result = <-resultChan want = &TabletHealth{ Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, @@ -102,23 +98,8 @@ func TestBasicHealthCheck(t *testing.T) { Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5}, MasterTermStartTime: 0, } - startTime = time.Now() - lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTime() - for { - // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for health update") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) - targetKey := hc.keyFromTarget(want.Target) - ths := hc.healthData[targetKey] - assert.NotEmpty(t, ths, "healthData is empty") - assert.True(t, want.DeepEqual(ths[tabletAlias].SimpleCopy()), "healthData contains wrong tabletHealth") + // create a context with timeout and select on it and channel + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) tcsl := hc.CacheStatus() tcslWant := TabletsCacheStatusList{{ @@ -132,8 +113,9 @@ func TestBasicHealthCheck(t *testing.T) { MasterTermStartTime: 0, }}, }} + // we can't use assert.Equal here because of the special way we want to compare equality assert.True(t, tcslWant.deepEqual(tcsl), "Incorrect cache status:\n Expected: %+v\n Actual: %+v", tcslWant[0], tcsl[0]) - // testChecksum(t, 3487343103, hc.stateChecksum()) + testChecksum(t, 3487343103, hc.stateChecksum()) // TabletType changed, should get both old and new event shr = &querypb.StreamHealthResponse{ @@ -155,20 +137,10 @@ func TestBasicHealthCheck(t *testing.T) { MasterTermStartTime: 10, } input <- shr - startTime = time.Now() - lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTime() - for { - // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for health update") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) - // testChecksum(t, 2773351292, hc.stateChecksum()) + result = <-resultChan + + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + testChecksum(t, 1780128002, hc.stateChecksum()) err := checkErrorCounter("k", "s", topodatapb.TabletType_MASTER, 0) require.NoError(t, err, "error checking error counter") @@ -189,20 +161,9 @@ func TestBasicHealthCheck(t *testing.T) { } input <- shr t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.3}}`) - startTime = time.Now() - lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTime() - for { - // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for health update") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) - // testChecksum(t, 2829991735, hc.stateChecksum()) + result = <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + testChecksum(t, 1027934207, hc.stateChecksum()) // HealthError shr = &querypb.StreamHealthResponse{ @@ -221,26 +182,14 @@ func TestBasicHealthCheck(t *testing.T) { } input <- shr t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3}}`) - startTime = time.Now() - lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTime() - for { - // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for health update") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong health data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + result = <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) testChecksum(t, 1027934207, hc.stateChecksum()) // unchanged // remove tablet hc.deleteConn(tablet) t.Logf(`hc.RemoveTablet({Host: "a", PortMap: {"vt": 1}})`) - assert.Nil(t, hc.healthByAlias[tabletAlias], "Wrong tablet health") testChecksum(t, 0, hc.stateChecksum()) } @@ -248,35 +197,28 @@ func TestBasicHealthCheck(t *testing.T) { func TestHealthCheckStreamError(t *testing.T) { ts := memorytopo.NewServer("cell") hc := createTestHc(ts) + defer hc.Close() + tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 - tabletAlias := topoproto.TabletAliasString(tablet.Alias) input := make(chan *querypb.StreamHealthResponse) + resultChan := hc.Subscribe() fc := createFakeConn(tablet, input) fc.errCh = make(chan error) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) + hc.AddTablet(tablet) t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. - want := &tabletHealthCheck{ + want := &TabletHealth{ Tablet: tablet, Target: &querypb.Target{}, Serving: false, MasterTermStartTime: 0, } - startTime := time.Now() - timeout := 2 * time.Second - for { - if hc.healthByAlias[tabletAlias] != nil { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for initial health") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + result := <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) // one tablet after receiving a StreamHealthResponse shr := &querypb.StreamHealthResponse{ @@ -285,7 +227,7 @@ func TestHealthCheckStreamError(t *testing.T) { TabletExternallyReparentedTimestamp: 0, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - want = &tabletHealthCheck{ + want = &TabletHealth{ Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, @@ -294,23 +236,12 @@ func TestHealthCheckStreamError(t *testing.T) { } input <- shr t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) - startTime = time.Now() - lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTime() - for { - // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for health update") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + result = <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) // Stream error fc.errCh <- fmt.Errorf("some stream error") - want = &tabletHealthCheck{ + want = &TabletHealth{ Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: false, @@ -318,56 +249,34 @@ func TestHealthCheckStreamError(t *testing.T) { MasterTermStartTime: 0, LastError: fmt.Errorf("some stream error"), } - startTime = time.Now() - lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTime() - for { - // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for health update") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) - - // close healthcheck - hc.Close() + result = <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) } func TestHealthCheckVerifiesTabletAlias(t *testing.T) { ts := memorytopo.NewServer("cell") - t.Logf("starting") + hc := createTestHc(ts) + defer hc.Close() + tablet := topo.NewTablet(1, "cell", "a") tablet.PortMap["vt"] = 1 - tabletAlias := topoproto.TabletAliasString(tablet.Alias) input := make(chan *querypb.StreamHealthResponse, 1) fc := createFakeConn(tablet, input) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) - hc := createTestHc(ts) + resultChan := hc.Subscribe() + hc.AddTablet(tablet) t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. - want := &tabletHealthCheck{ + want := &TabletHealth{ Tablet: tablet, Target: &querypb.Target{}, Serving: false, MasterTermStartTime: 0, } - startTime := time.Now() - timeout := 2 * time.Second - for { - if hc.healthByAlias[tabletAlias] != nil { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for initial health") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + result := <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) input <- &querypb.StreamHealthResponse{ Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, @@ -377,41 +286,16 @@ func TestHealthCheckVerifiesTabletAlias(t *testing.T) { RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - ticker := time.NewTicker(2 * time.Second) + ticker := time.NewTicker(1 * time.Second) select { case err := <-fc.cbErrCh: t.Logf("<-fc.cbErrCh: %v", err) - if prefix := "health stats mismatch"; !strings.HasPrefix(err.Error(), prefix) { - t.Fatalf("wrong error, got %v; want prefix %v", err, prefix) - } + assert.Contains(t, err.Error(), "health stats mismatch", "wrong error") + case <-resultChan: + require.Fail(t, "StreamHealth should have returned a health stats mismatch error") case <-ticker.C: - t.Fatalf("Timed out waiting for StreamHealth to return a health stats mismatch error") + require.Fail(t, "Timed out waiting for StreamHealth to return a health stats mismatch error") } - - input <- &querypb.StreamHealthResponse{ - Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, - TabletAlias: &topodatapb.TabletAlias{Uid: 1, Cell: "cell"}, - Serving: true, - TabletExternallyReparentedTimestamp: 10, - RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, - } - - startTime = time.Now() - lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTime() - for { - // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for health update") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) - - // close healthcheck - hc.Close() } // TestHealthCheckCloseWaitsForGoRoutines tests that Close() waits for all Go @@ -421,32 +305,23 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { hc := createTestHc(ts) tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 - tabletAlias := topoproto.TabletAliasString(tablet.Alias) input := make(chan *querypb.StreamHealthResponse, 1) createFakeConn(tablet, input) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) + resultChan := hc.Subscribe() + hc.AddTablet(tablet) t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. - want := &tabletHealthCheck{ + want := &TabletHealth{ Tablet: tablet, Target: &querypb.Target{}, Serving: false, MasterTermStartTime: 0, } - startTime := time.Now() - timeout := 2 * time.Second - for { - if hc.healthByAlias[tabletAlias] != nil { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for initial health") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + result := <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) // one tablet after receiving a StreamHealthResponse shr := &querypb.StreamHealthResponse{ @@ -455,7 +330,7 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { TabletExternallyReparentedTimestamp: 0, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - want = &tabletHealthCheck{ + want = &TabletHealth{ Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, @@ -464,19 +339,8 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { } input <- shr t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) - startTime = time.Now() - lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTime() - for { - // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for health update") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias]), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + result = <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) // Change input to distinguish between stats sent before and after Close(). shr.TabletExternallyReparentedTimestamp = 11 @@ -488,20 +352,32 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { // fail in some cases. input <- shr t.Logf(`input <- %v`, shr) + + select { + case result = <-resultChan: + assert.Nil(t, result, "healthCheck still running after Close(): listener received: %v but should not have been called", result) + case <-time.After(1 * time.Millisecond): + // No response after timeout. Success. + } + + hc.mu.Lock() + defer hc.mu.Unlock() assert.Nil(t, hc.healthByAlias, "health data should be nil") } func TestHealthCheckTimeout(t *testing.T) { ts := memorytopo.NewServer("cell") hc := createTestHc(ts) - timeout := 500 * time.Millisecond - hc.healthCheckTimeout = 2 * timeout + hc.healthCheckTimeout = 500 * time.Millisecond + defer hc.Close() + tablet := topo.NewTablet(0, "cell", "a") tablet.PortMap["vt"] = 1 - tabletAlias := topoproto.TabletAliasString(tablet.Alias) input := make(chan *querypb.StreamHealthResponse) fc := createFakeConn(tablet, input) t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) + resultChan := hc.Subscribe() + hc.AddTablet(tablet) t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. @@ -511,17 +387,8 @@ func TestHealthCheckTimeout(t *testing.T) { Serving: false, MasterTermStartTime: 0, } - startTime := time.Now() - for { - if hc.healthByAlias[tabletAlias] != nil { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for initial health") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong tabletHealth data:\n expected: %v\n got: %v", want, hc.healthByAlias[tabletAlias]) + result := <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) // one tablet after receiving a StreamHealthResponse shr := &querypb.StreamHealthResponse{ @@ -539,19 +406,8 @@ func TestHealthCheckTimeout(t *testing.T) { } input <- shr t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) - startTime = time.Now() - lastResponseTime := hc.healthByAlias[tabletAlias].lastResponseTime() - for { - // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for health update") - } - time.Sleep(10 * time.Microsecond) - } - assert.True(t, want.DeepEqual(hc.healthByAlias[tabletAlias].SimpleCopy()), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) + result = <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) if err := checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 0); err != nil { t.Errorf("%v", err) @@ -560,9 +416,9 @@ func TestHealthCheckTimeout(t *testing.T) { // wait for timeout period time.Sleep(hc.healthCheckTimeout + 100*time.Millisecond) t.Logf(`Sleep(1.1 * timeout)`) - res := hc.healthByAlias[tabletAlias] - if res.Serving { - t.Errorf(`tabletHealthCheck: %+v; want not serving`, res) + result = <-resultChan + if result.Serving { + t.Errorf(`tabletHealthCheck: %+v; want not serving`, result) } if err := checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 1); err != nil { @@ -578,9 +434,9 @@ func TestHealthCheckTimeout(t *testing.T) { time.Sleep(hc.healthCheckTimeout) t.Logf(`Sleep(timeout)`) - res = hc.healthByAlias[tabletAlias] - if res.Serving { - t.Errorf(`tabletHealthCheck: %+v; want not serving`, res) + result = <-resultChan + if result.Serving { + t.Errorf(`tabletHealthCheck: %+v; want not serving`, result) } if err := checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 2); err != nil { @@ -597,22 +453,8 @@ func TestHealthCheckTimeout(t *testing.T) { t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) // wait for the exponential backoff to wear off and health monitoring to resume. - startTime = time.Now() - lastResponseTime = hc.healthByAlias[tabletAlias].lastResponseTime() - for { - // Health has been updated - if hc.healthByAlias[tabletAlias].lastResponseTime() != lastResponseTime || time.Since(startTime) > timeout { - break - } - if time.Since(startTime) > timeout { - t.Fatal("Timed out waiting for health update") - } - time.Sleep(10 * time.Microsecond) - } - res = hc.healthByAlias[tabletAlias] - assert.True(t, want.DeepEqual(res.SimpleCopy()), "Wrong tabletHealth data:\n Expected: %v\n Actual: %v", want, hc.healthByAlias[tabletAlias]) - // close healthcheck - hc.Close() + result = <-resultChan + assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) } func TestTemplate(t *testing.T) { @@ -679,3 +521,86 @@ func tabletDialer(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (quer func createTestHc(ts *topo.Server) *HealthCheckImpl { return NewHealthCheck(context.Background(), 1*time.Millisecond, time.Hour, ts, "cell", nil).(*HealthCheckImpl) } + +type fakeConn struct { + queryservice.QueryService + tablet *topodatapb.Tablet + // If fixedResult is set, the channels are not used. + fixedResult *querypb.StreamHealthResponse + // hcChan should be an unbuffered channel which holds the tablet's next health response. + hcChan chan *querypb.StreamHealthResponse + // errCh is either an unbuffered channel which holds the stream error to return, or nil. + errCh chan error + // cbErrCh is a channel which receives errors returned from the supplied callback. + cbErrCh chan error + + mu sync.Mutex + canceled bool +} + +func createFakeConn(tablet *topodatapb.Tablet, c chan *querypb.StreamHealthResponse) *fakeConn { + key := TabletToMapKey(tablet) + conn := &fakeConn{ + QueryService: fakes.ErrorQueryService, + tablet: tablet, + hcChan: c, + cbErrCh: make(chan error, 1), + } + connMap[key] = conn + return conn +} + +// StreamHealth implements queryservice.QueryService. +func (fc *fakeConn) StreamHealth(ctx context.Context, callback func(shr *querypb.StreamHealthResponse) error) error { + if fc.fixedResult != nil { + return callback(fc.fixedResult) + } + for { + select { + case shr := <-fc.hcChan: + if err := callback(shr); err != nil { + if err == io.EOF { + return nil + } + select { + case fc.cbErrCh <- err: + case <-ctx.Done(): + } + return err + } + case err := <-fc.errCh: + return err + case <-ctx.Done(): + fc.mu.Lock() + fc.canceled = true + fc.mu.Unlock() + return nil + } + } +} + +func (fc *fakeConn) isCanceled() bool { + fc.mu.Lock() + defer fc.mu.Unlock() + return fc.canceled +} + +func (fc *fakeConn) resetCanceledFlag() { + fc.mu.Lock() + defer fc.mu.Unlock() + fc.canceled = false +} + +func checkErrorCounter(keyspace, shard string, tabletType topodatapb.TabletType, want int64) error { + statsKey := []string{keyspace, shard, topoproto.TabletTypeLString(tabletType)} + name := strings.Join(statsKey, ".") + got, ok := hcErrorCounters.Counts()[name] + if !ok { + return fmt.Errorf("hcErrorCounters not correctly initialized") + } + if got != want { + return fmt.Errorf("wrong value for hcErrorCounters got = %v, want = %v", got, want) + } + + return nil +} diff --git a/go/vt/discovery/legacy_healthcheck_flaky_test.go b/go/vt/discovery/legacy_healthcheck_flaky_test.go index 51ed08e7cab..8d28ad60abd 100644 --- a/go/vt/discovery/legacy_healthcheck_flaky_test.go +++ b/go/vt/discovery/legacy_healthcheck_flaky_test.go @@ -21,10 +21,8 @@ import ( "flag" "fmt" "html/template" - "io" "reflect" "strings" - "sync" "testing" "time" @@ -32,7 +30,6 @@ import ( "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/status" "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vttablet/queryservice" "vitess.io/vitess/go/vt/vttablet/queryservice/fakes" "vitess.io/vitess/go/vt/vttablet/tabletconn" @@ -657,34 +654,6 @@ func (l *listener) StatsUpdate(ts *LegacyTabletStats) { l.output <- ts } -type fakeConn struct { - queryservice.QueryService - tablet *topodatapb.Tablet - // If fixedResult is set, the channels are not used. - fixedResult *querypb.StreamHealthResponse - // hcChan should be an unbuffered channel which holds the tablet's next health response. - hcChan chan *querypb.StreamHealthResponse - // errCh is either an unbuffered channel which holds the stream error to return, or nil. - errCh chan error - // cbErrCh is a channel which receives errors returned from the supplied callback. - cbErrCh chan error - - mu sync.Mutex - canceled bool -} - -func createFakeConn(tablet *topodatapb.Tablet, c chan *querypb.StreamHealthResponse) *fakeConn { - key := TabletToMapKey(tablet) - conn := &fakeConn{ - QueryService: fakes.ErrorQueryService, - tablet: tablet, - hcChan: c, - cbErrCh: make(chan error, 1), - } - connMap[key] = conn - return conn -} - func createFixedHealthConn(tablet *topodatapb.Tablet, fixedResult *querypb.StreamHealthResponse) *fakeConn { key := TabletToMapKey(tablet) conn := &fakeConn{ @@ -703,58 +672,3 @@ func discoveryDialer(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (q } return nil, fmt.Errorf("tablet %v not found", key) } - -// StreamHealth implements queryservice.QueryService. -func (fc *fakeConn) StreamHealth(ctx context.Context, callback func(shr *querypb.StreamHealthResponse) error) error { - if fc.fixedResult != nil { - return callback(fc.fixedResult) - } - for { - select { - case shr := <-fc.hcChan: - if err := callback(shr); err != nil { - if err == io.EOF { - return nil - } - select { - case fc.cbErrCh <- err: - case <-ctx.Done(): - } - return err - } - case err := <-fc.errCh: - return err - case <-ctx.Done(): - fc.mu.Lock() - fc.canceled = true - fc.mu.Unlock() - return nil - } - } -} - -func (fc *fakeConn) isCanceled() bool { - fc.mu.Lock() - defer fc.mu.Unlock() - return fc.canceled -} - -func (fc *fakeConn) resetCanceledFlag() { - fc.mu.Lock() - defer fc.mu.Unlock() - fc.canceled = false -} - -func checkErrorCounter(keyspace, shard string, tabletType topodatapb.TabletType, want int64) error { - statsKey := []string{keyspace, shard, topoproto.TabletTypeLString(tabletType)} - name := strings.Join(statsKey, ".") - got, ok := hcErrorCounters.Counts()[name] - if !ok { - return fmt.Errorf("hcErrorCounters not correctly initialized") - } - if got != want { - return fmt.Errorf("wrong value for hcErrorCounters got = %v, want = %v", got, want) - } - - return nil -} diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 8120e01d695..0050d782dce 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -128,21 +128,16 @@ func (th *tabletHealthCheck) String() string { th.Tablet, th.Target, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) } -func (th *tabletHealthCheck) lastResponseTime() time.Time { - th.mu.Lock() - defer th.mu.Unlock() - res := th.lastResponseTimestamp - return res -} - // SimpleCopy returns a TabletHealth with all the necessary fields copied from tabletHealthCheck. // Note that this is not a deep copy because we point to the same underlying RealtimeStats. // That is fine because the RealtimeStats object is never changed after creation. func (th *tabletHealthCheck) SimpleCopy() *TabletHealth { - // we have to explicitly create a new object rather than relying on assignment to make a copy for us - // the following doesn't work for synchronized objects - // t := *th - // return &t + th.mu.Lock() + defer th.mu.Unlock() + return th.simpleCopyLocked() +} + +func (th *tabletHealthCheck) simpleCopyLocked() *TabletHealth { return &TabletHealth{ Conn: th.Conn, Tablet: th.Tablet, @@ -203,33 +198,42 @@ func (th *tabletHealthCheck) setServingState(serving bool, reason string) { // stream streams healthcheck responses to callback. func (th *tabletHealthCheck) stream(ctx context.Context, callback func(*query.StreamHealthResponse) error) error { + conn := th.getConnection() + if conn == nil { + // This signals the caller to retry + return nil + } + err := conn.StreamHealth(ctx, callback) + if err != nil { + // Depending on the specific error the caller can take action + th.closeConnection(ctx, err) + } + return err +} + +func (th *tabletHealthCheck) getConnection() queryservice.QueryService { th.mu.Lock() + defer th.mu.Unlock() if th.Conn == nil { conn, err := tabletconn.GetDialer()(th.Tablet, grpcclient.FailFast(true)) if err != nil { th.LastError = err - th.mu.Unlock() return nil } th.Conn = conn th.LastError = nil } - conn := th.Conn - th.mu.Unlock() + return th.Conn +} - err := conn.StreamHealth(ctx, callback) - if err != nil { - th.mu.Lock() - log.Warningf("tablet %v healthcheck stream error: %v", th.Tablet.Alias, err) - th.setServingState(false, err.Error()) - th.LastError = err - th.Conn.Close(ctx) - th.Conn = nil - // signal that healthCheck is now up-to-date - th.lastResponseTimestamp = time.Now() - th.mu.Unlock() - } - return err +func (th *tabletHealthCheck) closeConnection(ctx context.Context, err error) { + th.mu.Lock() + defer th.mu.Unlock() + log.Warningf("tablet %v healthcheck stream error: %v", th.Tablet.Alias, err) + th.setServingState(false, err.Error()) + th.LastError = err + _ = th.Conn.Close(ctx) + th.Conn = nil } // processResponse reads one health check response, and updates health @@ -259,7 +263,6 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str // However, we defer it until the next topo refresh informs us of the change because that is // the only way to discover the new host/port return vterrors.New(vtrpc.Code_FAILED_PRECONDITION, fmt.Sprintf("health stats mismatch, tablet %+v alias does not match response alias %v", th.Tablet, shr.TabletAlias)) - // TODO(deepthi): delete healthcheck } th.mu.Lock() @@ -297,6 +300,7 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str // Update our record th.mu.Lock() defer th.mu.Unlock() + th.lastResponseTimestamp = time.Now() th.Target = shr.Target th.MasterTermStartTime = shr.TabletExternallyReparentedTimestamp th.Stats = shr.RealtimeStats @@ -335,13 +339,13 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str } } } + + result := th.simpleCopyLocked() + hc.broadcast(result) // and notify downstream for master change - if shr.Target.TabletType == topodata.TabletType_MASTER { - if hc.masterCallback != nil { - hc.masterCallback(th.SimpleCopy()) - } + if shr.Target.TabletType == topodata.TabletType_MASTER && hc.masterCallback != nil { + hc.masterCallback(result) } - // signal that healthCheck is now up-to-date - th.lastResponseTimestamp = time.Now() + return nil } diff --git a/go/vt/discovery/tablets_cache_status.go b/go/vt/discovery/tablets_cache_status.go index 7261bb9708e..46d2fd4ba4d 100644 --- a/go/vt/discovery/tablets_cache_status.go +++ b/go/vt/discovery/tablets_cache_status.go @@ -73,7 +73,7 @@ func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML { } else { extra = fmt.Sprintf(" (RepLag: %v)", ts.Stats.SecondsBehindMaster) } - name := ts.GetTabletHostPort() + name := topoproto.TabletAliasString(ts.Tablet.Alias) tLinks = append(tLinks, fmt.Sprintf(`%v%v`, ts.getTabletDebugURL(), color, name, extra)) } return template.HTML(strings.Join(tLinks, "
")) From 4b85e4ba8910b9071850bb34140e7883e8d82888 Mon Sep 17 00:00:00 2001 From: deepthi Date: Wed, 6 May 2020 22:44:50 -0700 Subject: [PATCH 21/39] healthcheck: ScatterConn should return the correct cache status based on gateway_implementation Signed-off-by: deepthi --- go/vt/vtgate/executor.go | 68 +++++++++++++++++++++++++----------- go/vt/vtgate/scatter_conn.go | 4 +-- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 07bf0dfcbc8..ffae58d4625 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -841,28 +841,56 @@ func (e *Executor) handleShow(ctx context.Context, safeSession *SafeSession, sql }, nil case "vitess_tablets": var rows [][]sqltypes.Value - stats := e.scatterConn.GetLegacyHealthCheckCacheStatus() - for _, s := range stats { - for _, ts := range s.TabletsStats { - state := "SERVING" - if !ts.Serving { - state = "NOT_SERVING" + if *GatewayImplementation == GatewayImplementationDiscovery { + stats := e.scatterConn.GetLegacyHealthCheckCacheStatus() + for _, s := range stats { + for _, ts := range s.TabletsStats { + state := "SERVING" + if !ts.Serving { + state = "NOT_SERVING" + } + mtst := ts.Tablet.MasterTermStartTime + mtstStr := "" + if mtst != nil && mtst.Seconds > 0 { + mtstStr = logutil.ProtoToTime(ts.Tablet.MasterTermStartTime).Format(time.RFC3339) + } + rows = append(rows, buildVarCharRow( + s.Cell, + s.Target.Keyspace, + s.Target.Shard, + ts.Target.TabletType.String(), + state, + topoproto.TabletAliasString(ts.Tablet.Alias), + ts.Tablet.Hostname, + mtstStr, + )) } - mtst := ts.Tablet.MasterTermStartTime - mtstStr := "" - if mtst != nil && mtst.Seconds > 0 { - mtstStr = logutil.ProtoToTime(ts.Tablet.MasterTermStartTime).Format(time.RFC3339) + } + } + if *GatewayImplementation == tabletGatewayImplementation { + stats := e.scatterConn.GetLegacyHealthCheckCacheStatus() + for _, s := range stats { + for _, ts := range s.TabletsStats { + state := "SERVING" + if !ts.Serving { + state = "NOT_SERVING" + } + mtst := ts.Tablet.MasterTermStartTime + mtstStr := "" + if mtst != nil && mtst.Seconds > 0 { + mtstStr = logutil.ProtoToTime(ts.Tablet.MasterTermStartTime).Format(time.RFC3339) + } + rows = append(rows, buildVarCharRow( + s.Cell, + s.Target.Keyspace, + s.Target.Shard, + ts.Target.TabletType.String(), + state, + topoproto.TabletAliasString(ts.Tablet.Alias), + ts.Tablet.Hostname, + mtstStr, + )) } - rows = append(rows, buildVarCharRow( - s.Cell, - s.Target.Keyspace, - s.Target.Shard, - ts.Target.TabletType.String(), - state, - topoproto.TabletAliasString(ts.Tablet.Alias), - ts.Tablet.Hostname, - mtstStr, - )) } } return &sqltypes.Result{ diff --git a/go/vt/vtgate/scatter_conn.go b/go/vt/vtgate/scatter_conn.go index 6e2095acb0a..663bf20ac41 100644 --- a/go/vt/vtgate/scatter_conn.go +++ b/go/vt/vtgate/scatter_conn.go @@ -451,10 +451,10 @@ func (stc *ScatterConn) GetLegacyHealthCheckCacheStatus() discovery.LegacyTablet } // GetHealthCheckCacheStatus returns a displayable version of the HealthCheck cache. -func (stc *ScatterConn) GetHealthCheckCacheStatus() TabletCacheStatusList { +func (stc *ScatterConn) GetHealthCheckCacheStatus() discovery.TabletsCacheStatusList { gw, ok := stc.gateway.(*TabletGateway) if ok { - return gw.CacheStatus() + return gw.HealthCheck().CacheStatus() } return nil } From b2f8c8965ab1754c3a9ee2007bab8db6e37d4b2a Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 7 May 2020 16:53:31 -0700 Subject: [PATCH 22/39] healthcheck: topology_watcher unit tests Signed-off-by: deepthi --- go/vt/discovery/fake_healthcheck.go | 214 ++++++++ go/vt/discovery/healthcheck.go | 184 +------ .../discovery/legacy_topology_watcher_test.go | 19 - go/vt/discovery/topology_watcher.go | 196 +++++++- go/vt/discovery/topology_watcher_test.go | 460 ++++++++++++++++++ 5 files changed, 864 insertions(+), 209 deletions(-) create mode 100644 go/vt/discovery/fake_healthcheck.go create mode 100644 go/vt/discovery/topology_watcher_test.go diff --git a/go/vt/discovery/fake_healthcheck.go b/go/vt/discovery/fake_healthcheck.go new file mode 100644 index 00000000000..460aa5a934d --- /dev/null +++ b/go/vt/discovery/fake_healthcheck.go @@ -0,0 +1,214 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "sort" + "sync" + + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vttablet/queryservice" + "vitess.io/vitess/go/vt/vttablet/sandboxconn" + + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +// This file contains the definitions for a FakeHealthCheck class to +// simulate a LegacyHealthCheck module. Note it is not in a sub-package because +// otherwise it couldn't be used in this package's tests because of +// circular dependencies. + +// NewFakeHealthCheck returns the fake healthcheck object. +func NewFakeHealthCheck() *FakeHealthCheck { + return &FakeHealthCheck{ + items: make(map[string]*fhcItem), + } +} + +// FakeHealthCheck implements discovery.LegacyHealthCheck. +type FakeHealthCheck struct { + // mu protects the items map + mu sync.RWMutex + items map[string]*fhcItem +} + +type fhcItem struct { + ts *TabletHealth + conn queryservice.QueryService +} + +// +// discovery.LegacyHealthCheck interface methods +// + +// RegisterStats is not implemented. +func (fhc *FakeHealthCheck) RegisterStats() { +} + +// WaitForInitialStatsUpdates is not implemented. +func (fhc *FakeHealthCheck) WaitForInitialStatsUpdates() { +} + +// AddTablet adds the tablet and calls the listener. +func (fhc *FakeHealthCheck) AddTablet(tablet *topodatapb.Tablet) { + key := TabletToMapKey(tablet) + item := &fhcItem{ + ts: &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{ + Keyspace: tablet.Keyspace, + Shard: tablet.Shard, + TabletType: tablet.Type, + }, + Serving: true, + Stats: &querypb.RealtimeStats{}, + }, + } + + fhc.mu.Lock() + defer fhc.mu.Unlock() + fhc.items[key] = item +} + +// RemoveTablet removes the tablet. +func (fhc *FakeHealthCheck) RemoveTablet(tablet *topodatapb.Tablet) { + fhc.mu.Lock() + defer fhc.mu.Unlock() + key := TabletToMapKey(tablet) + item, ok := fhc.items[key] + if !ok { + return + } + // Make sure the key still corresponds to the tablet we want to delete. + // If it doesn't match, we should do nothing. The tablet we were asked to + // delete is already gone, and some other tablet is using the key + // (host:port) that the original tablet used to use, which is fine. + if !topoproto.TabletAliasEqual(tablet.Alias, item.ts.Tablet.Alias) { + return + } + delete(fhc.items, key) +} + +// ReplaceTablet removes the old tablet and adds the new. +func (fhc *FakeHealthCheck) ReplaceTablet(old, new *topodatapb.Tablet) { + fhc.RemoveTablet(old) + fhc.AddTablet(new) +} + +// GetConnection returns the TabletConn of the given tablet. +func (fhc *FakeHealthCheck) GetConnection(key string) queryservice.QueryService { + fhc.mu.RLock() + defer fhc.mu.RUnlock() + if item := fhc.items[key]; item != nil { + return item.conn + } + return nil +} + +// CacheStatus returns the status for each tablet +func (fhc *FakeHealthCheck) CacheStatus() TabletsCacheStatusList { + fhc.mu.Lock() + defer fhc.mu.Unlock() + + stats := make(TabletsCacheStatusList, 0, len(fhc.items)) + for _, item := range fhc.items { + stats = append(stats, &TabletsCacheStatus{ + Cell: "FakeCell", + Target: item.ts.Target, + TabletsStats: TabletStatsList{item.ts}, + }) + } + sort.Sort(stats) + return stats +} + +// Close is not implemented. +func (fhc *FakeHealthCheck) Close() error { + return nil +} + +// +// Management methods +// + +// Reset cleans up the internal state. +func (fhc *FakeHealthCheck) Reset() { + fhc.mu.Lock() + defer fhc.mu.Unlock() + + fhc.items = make(map[string]*fhcItem) +} + +// AddFakeTablet inserts a fake entry into FakeHealthCheck. +// The Tablet can be talked to using the provided connection. +// The Listener is called, as if AddTablet had been called. +// For flexibility the connection is created via a connFactory callback +func (fhc *FakeHealthCheck) AddFakeTablet(cell, host string, port int32, keyspace, shard string, tabletType topodatapb.TabletType, serving bool, reparentTS int64, err error, connFactory func(*topodatapb.Tablet) queryservice.QueryService) queryservice.QueryService { + t := topo.NewTablet(0, cell, host) + t.Keyspace = keyspace + t.Shard = shard + t.Type = tabletType + t.PortMap["vt"] = port + key := TabletToMapKey(t) + + fhc.mu.Lock() + defer fhc.mu.Unlock() + item := fhc.items[key] + if item == nil { + item = &fhcItem{ + ts: &TabletHealth{ + Tablet: t, + }, + } + fhc.items[key] = item + } + item.ts.Target = &querypb.Target{ + Keyspace: keyspace, + Shard: shard, + TabletType: tabletType, + } + item.ts.Serving = serving + item.ts.MasterTermStartTime = reparentTS + item.ts.Stats = &querypb.RealtimeStats{} + item.ts.LastError = err + conn := connFactory(t) + item.conn = conn + + return conn +} + +// AddTestTablet adds a fake tablet for tests using the SandboxConn and returns +// the fake connection +func (fhc *FakeHealthCheck) AddTestTablet(cell, host string, port int32, keyspace, shard string, tabletType topodatapb.TabletType, serving bool, reparentTS int64, err error) *sandboxconn.SandboxConn { + conn := fhc.AddFakeTablet(cell, host, port, keyspace, shard, tabletType, serving, reparentTS, err, func(tablet *topodatapb.Tablet) queryservice.QueryService { + return sandboxconn.NewSandboxConn(tablet) + }) + return conn.(*sandboxconn.SandboxConn) +} + +// GetAllTablets returns all the tablets we have. +func (fhc *FakeHealthCheck) GetAllTablets() map[string]*topodatapb.Tablet { + res := make(map[string]*topodatapb.Tablet) + fhc.mu.RLock() + defer fhc.mu.RUnlock() + for key, t := range fhc.items { + res[key] = t.ts.Tablet + } + return res +} diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 09c7ae49c3d..1ef7c05a136 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -147,6 +147,17 @@ func init() { flag.Var(&KeyspacesToWatch, "keyspaces_to_watch", "Specifies which keyspaces this vtgate should have access to while routing queries or accessing the vschema") } +// TabletRecorder is a sub interface of HealthCheck. +// It is separated out to enable unit testing. +type TabletRecorder interface { + // AddTablet adds the tablet. + AddTablet(tablet *topodata.Tablet) + // RemoveTablet removes the tablet. + RemoveTablet(tablet *topodata.Tablet) + // ReplaceTablet does an AddTablet and RemoveTablet in one call, effectively replacing the old tablet with the new. + ReplaceTablet(old, new *topodata.Tablet) +} + // HealthCheck defines the interface of health checking module. // The goal of this object is to maintain a StreamHealth RPC // to a lot of tablets. Tablets are added / removed by calling the @@ -164,12 +175,6 @@ type HealthCheck interface { GetHealthyTabletStats(target *query.Target) []*TabletHealth // WaitForAllServingTablets allows vtgate to wait for all tablets to be serving before accepting requests WaitForAllServingTablets(ctx context.Context, targets []*query.Target) error - // AddTablet adds the tablet. - AddTablet(tablet *topodata.Tablet) - // RemoveTablet removes the tablet. - RemoveTablet(tablet *topodata.Tablet) - // ReplaceTablet does an AddTablet and RemoveTablet in one call, effectively replacing the old tablet with the new. - ReplaceTablet(old, new *topodata.Tablet) } // HealthCheckImpl performs health checking and stores the results. @@ -205,8 +210,6 @@ type HealthCheckImpl struct { // used to inform vtgate buffer when new master is detected // TODO: replace this with synchronizing over a condition variable masterCallback func(health *TabletHealth) - // cellAliases is a cache of cell aliases - cellAliases map[string]string // mutex to protect subscribers subMu sync.Mutex // subscribers @@ -291,9 +294,9 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur } filter = fbs } else if len(KeyspacesToWatch) > 0 { - filter = NewFilterByKeyspace(c, KeyspacesToWatch) + filter = NewFilterByKeyspace(KeyspacesToWatch) } - topoWatchers = append(topoWatchers, NewCellTabletsWatcher(ctx, topoServer, filter, c, *RefreshInterval, *RefreshKnownTablets, *TopoReadConcurrency)) + topoWatchers = append(topoWatchers, NewCellTabletsWatcher(ctx, topoServer, hc, filter, c, *RefreshInterval, *RefreshKnownTablets, *TopoReadConcurrency)) } hc.topoWatchers = topoWatchers @@ -303,171 +306,12 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur // start the topo watches here for _, tw := range hc.topoWatchers { - go hc.watchTopo(tw) + go tw.Start() } return hc } -func (hc *HealthCheckImpl) watchTopo(tw *TopologyWatcher) { - tw.wg.Add(1) - defer tw.wg.Done() - ticker := time.NewTicker(tw.refreshInterval) - defer ticker.Stop() - for { - hc.loadTablets(tw) - select { - case <-tw.ctx.Done(): - return - case <-ticker.C: - } - } -} - -func (hc *HealthCheckImpl) loadTablets(tw *TopologyWatcher) { - var wg sync.WaitGroup - newTablets := make(map[string]*tabletInfo) - - // first get the list of relevant tabletAliases - tabletAliases, err := tw.getTablets(tw) - topologyWatcherOperations.Add(topologyWatcherOpListTablets, 1) - if err != nil { - topologyWatcherErrors.Add(topologyWatcherOpListTablets, 1) - select { - case <-tw.ctx.Done(): - return - default: - } - log.Errorf("cannot get tablets for cell: %v: %v", tw.cell, err) - return - } - - // Accumulate a list of all known alias strings to use later - // when sorting - tabletAliasStrs := make([]string, 0, len(tabletAliases)) - - tw.mu.Lock() - for _, tAlias := range tabletAliases { - aliasStr := topoproto.TabletAliasString(tAlias) - tabletAliasStrs = append(tabletAliasStrs, aliasStr) - - if !tw.refreshKnownTablets { - // we already have a tabletInfo for this and the flag tells us to not refresh - if val, ok := tw.tablets[aliasStr]; ok { - newTablets[aliasStr] = val - continue - } - } - - wg.Add(1) - go func(alias *topodata.TabletAlias) { - defer wg.Done() - tw.sem <- 1 // Wait for active queue to drain. - tablet, err := tw.topoServer.GetTablet(tw.ctx, alias) - topologyWatcherOperations.Add(topologyWatcherOpGetTablet, 1) - <-tw.sem // Done; enable next request to run - if err != nil { - topologyWatcherErrors.Add(topologyWatcherOpGetTablet, 1) - select { - case <-tw.ctx.Done(): - return - default: - } - log.Errorf("cannot get tablet for alias %v: %v", alias, err) - return - } - if !(hc.isTabletInCell(tablet.Tablet) && (tw.tabletFilter == nil || tw.tabletFilter.IsIncluded(tablet.Tablet))) { - log.Errorf("loadTablets skipping tablet: %#v", tablet.Tablet) - return - } - tw.mu.Lock() - aliasStr := topoproto.TabletAliasString(alias) - newTablets[aliasStr] = &tabletInfo{ - alias: aliasStr, - tablet: tablet.Tablet, - } - tw.mu.Unlock() - }(tAlias) - } - - tw.mu.Unlock() - wg.Wait() - tw.mu.Lock() - - for alias, newVal := range newTablets { - // trust the alias from topo and add it if it doesn't exist - if val, ok := tw.tablets[alias]; !ok { - hc.AddTablet(newVal.tablet) - topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1) - } else { - // check if the host and port have changed. If yes, replace tablet - oldKey := TabletToMapKey(val.tablet) - newKey := TabletToMapKey(newVal.tablet) - if oldKey != newKey { - // This is the case where the same tablet alias is now reporting - // a different address key. - hc.ReplaceTablet(val.tablet, newVal.tablet) - topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) - } - } - } - - for _, val := range tw.tablets { - if _, ok := newTablets[val.alias]; !ok { - hc.RemoveTablet(val.tablet) - topologyWatcherOperations.Add(topologyWatcherOpRemoveTablet, 1) - } - } - tw.tablets = newTablets - if !tw.firstLoadDone { - tw.firstLoadDone = true - close(tw.firstLoadChan) - } - - // iterate through the tablets in a stable order and compute a - // checksum of the tablet map - sort.Strings(tabletAliasStrs) - var buf bytes.Buffer - for _, alias := range tabletAliasStrs { - _, ok := tw.tablets[alias] - if ok { - buf.WriteString(alias) - } - } - tw.topoChecksum = crc32.ChecksumIEEE(buf.Bytes()) - tw.lastRefresh = time.Now() - - tw.mu.Unlock() - -} - -func (hc *HealthCheckImpl) getAliasByCell(cell string) string { - hc.mu.Lock() - defer hc.mu.Unlock() - - if alias, ok := hc.cellAliases[cell]; ok { - return alias - } - - alias := topo.GetAliasByCell(context.Background(), hc.ts, cell) - hc.cellAliases[cell] = alias - - return alias -} - -func (hc *HealthCheckImpl) isTabletInCell(tablet *topodata.Tablet) bool { - if tablet.Type == topodata.TabletType_MASTER { - return true - } - if tablet.Alias.Cell == hc.cell { - return true - } - if hc.getAliasByCell(tablet.Alias.Cell) == hc.getAliasByCell(hc.cell) { - return true - } - return false -} - // RegisterStats registers the connection counts stats func (hc *HealthCheckImpl) RegisterStats() { stats.NewGaugeDurationFunc( diff --git a/go/vt/discovery/legacy_topology_watcher_test.go b/go/vt/discovery/legacy_topology_watcher_test.go index 06941bccfcb..ad20e104bed 100644 --- a/go/vt/discovery/legacy_topology_watcher_test.go +++ b/go/vt/discovery/legacy_topology_watcher_test.go @@ -412,25 +412,6 @@ func TestLegacyFilterByShard(t *testing.T) { } } -var ( - testFilterByKeyspace = []struct { - keyspace string - expected bool - }{ - {"ks1", true}, - {"ks2", true}, - {"ks3", false}, - {"ks4", true}, - {"ks5", true}, - {"ks6", false}, - {"ks7", false}, - } - testKeyspacesToWatch = []string{"ks1", "ks2", "ks4", "ks5"} - testCell = "testCell" - testShard = "testShard" - testHostName = "testHostName" -) - func TestLegacyFilterByKeyspace(t *testing.T) { hc := NewFakeLegacyHealthCheck() tr := NewLegacyFilterByKeyspace(hc, testKeyspacesToWatch) diff --git a/go/vt/discovery/topology_watcher.go b/go/vt/discovery/topology_watcher.go index 9b01befd1bb..a4d92af915a 100644 --- a/go/vt/discovery/topology_watcher.go +++ b/go/vt/discovery/topology_watcher.go @@ -17,11 +17,16 @@ limitations under the License. package discovery import ( + "bytes" "fmt" + "hash/crc32" + "sort" "strings" "sync" "time" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/key" "golang.org/x/net/context" @@ -29,7 +34,7 @@ import ( "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/log" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo" ) @@ -51,7 +56,7 @@ var ( // tabletInfo is used internally by the TopologyWatcher class type tabletInfo struct { alias string - tablet *topodatapb.Tablet + tablet *topodata.Tablet } // TopologyWatcher polls tablet from a configurable set of tablets @@ -60,11 +65,12 @@ type tabletInfo struct { type TopologyWatcher struct { // set at construction time topoServer *topo.Server + tabletRecorder TabletRecorder tabletFilter TabletFilter cell string refreshInterval time.Duration refreshKnownTablets bool - getTablets func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) + getTablets func(tw *TopologyWatcher) ([]*topodata.TabletAlias, error) sem chan int ctx context.Context cancelFunc context.CancelFunc @@ -83,13 +89,16 @@ type TopologyWatcher struct { firstLoadDone bool // firstLoadChan is closed when the initial loading of topology data is done. firstLoadChan chan struct{} + // cellAliases is a cache of cell aliases + cellAliases map[string]string } // NewTopologyWatcher returns a TopologyWatcher that monitors all // the tablets in a cell, and starts refreshing. -func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, filter TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error)) *TopologyWatcher { +func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, filter TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int, getTablets func(tw *TopologyWatcher) ([]*topodata.TabletAlias, error)) *TopologyWatcher { tw := &TopologyWatcher{ topoServer: topoServer, + tabletRecorder: tr, tabletFilter: filter, cell: cell, refreshInterval: refreshInterval, @@ -108,21 +117,25 @@ func NewTopologyWatcher(ctx context.Context, topoServer *topo.Server, filter Tab // NewCellTabletsWatcher returns a TopologyWatcher that monitors all // the tablets in a cell, and starts refreshing. -func NewCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, f TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *TopologyWatcher { - return NewTopologyWatcher(ctx, topoServer, f, cell, refreshInterval, refreshKnownTablets, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodatapb.TabletAlias, error) { +func NewCellTabletsWatcher(ctx context.Context, topoServer *topo.Server, tr TabletRecorder, f TabletFilter, cell string, refreshInterval time.Duration, refreshKnownTablets bool, topoReadConcurrency int) *TopologyWatcher { + return NewTopologyWatcher(ctx, topoServer, tr, f, cell, refreshInterval, refreshKnownTablets, topoReadConcurrency, func(tw *TopologyWatcher) ([]*topodata.TabletAlias, error) { return tw.topoServer.GetTabletsByCell(ctx, tw.cell) }) } -// WaitForInitialTopology waits until the watcher reads all of the topology data -// for the first time and transfers the information to LegacyTabletRecorder via its -// AddTablet() method. -func (tw *TopologyWatcher) WaitForInitialTopology() error { - select { - case <-tw.ctx.Done(): - return tw.ctx.Err() - case <-tw.firstLoadChan: - return nil +// Start starts the topology watcher +func (tw *TopologyWatcher) Start() { + tw.wg.Add(1) + defer tw.wg.Done() + ticker := time.NewTicker(tw.refreshInterval) + defer ticker.Stop() + for { + tw.loadTablets() + select { + case <-tw.ctx.Done(): + return + case <-ticker.C: + } } } @@ -133,6 +146,149 @@ func (tw *TopologyWatcher) Stop() { tw.wg.Wait() } +func (tw *TopologyWatcher) loadTablets() { + var wg sync.WaitGroup + newTablets := make(map[string]*tabletInfo) + + // first get the list of relevant tabletAliases + tabletAliases, err := tw.getTablets(tw) + topologyWatcherOperations.Add(topologyWatcherOpListTablets, 1) + if err != nil { + topologyWatcherErrors.Add(topologyWatcherOpListTablets, 1) + select { + case <-tw.ctx.Done(): + return + default: + } + log.Errorf("cannot get tablets for cell: %v: %v", tw.cell, err) + return + } + + // Accumulate a list of all known alias strings to use later + // when sorting + tabletAliasStrs := make([]string, 0, len(tabletAliases)) + + tw.mu.Lock() + for _, tAlias := range tabletAliases { + aliasStr := topoproto.TabletAliasString(tAlias) + tabletAliasStrs = append(tabletAliasStrs, aliasStr) + + if !tw.refreshKnownTablets { + // we already have a tabletInfo for this and the flag tells us to not refresh + if val, ok := tw.tablets[aliasStr]; ok { + newTablets[aliasStr] = val + continue + } + } + + wg.Add(1) + go func(alias *topodata.TabletAlias) { + defer wg.Done() + tw.sem <- 1 // Wait for active queue to drain. + tablet, err := tw.topoServer.GetTablet(tw.ctx, alias) + topologyWatcherOperations.Add(topologyWatcherOpGetTablet, 1) + <-tw.sem // Done; enable next request to run + if err != nil { + topologyWatcherErrors.Add(topologyWatcherOpGetTablet, 1) + select { + case <-tw.ctx.Done(): + return + default: + } + log.Errorf("cannot get tablet for alias %v: %v", alias, err) + return + } + if !(tw.isTabletInCell(tablet.Tablet) && (tw.tabletFilter == nil || tw.tabletFilter.IsIncluded(tablet.Tablet))) { + return + } + tw.mu.Lock() + aliasStr := topoproto.TabletAliasString(alias) + newTablets[aliasStr] = &tabletInfo{ + alias: aliasStr, + tablet: tablet.Tablet, + } + tw.mu.Unlock() + }(tAlias) + } + + tw.mu.Unlock() + wg.Wait() + tw.mu.Lock() + + for alias, newVal := range newTablets { + // trust the alias from topo and add it if it doesn't exist + if val, ok := tw.tablets[alias]; !ok { + tw.tabletRecorder.AddTablet(newVal.tablet) + topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1) + } else { + // check if the host and port have changed. If yes, replace tablet + oldKey := TabletToMapKey(val.tablet) + newKey := TabletToMapKey(newVal.tablet) + if oldKey != newKey { + // This is the case where the same tablet alias is now reporting + // a different address key. + tw.tabletRecorder.ReplaceTablet(val.tablet, newVal.tablet) + topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1) + } + } + } + + for _, val := range tw.tablets { + if _, ok := newTablets[val.alias]; !ok { + tw.tabletRecorder.RemoveTablet(val.tablet) + topologyWatcherOperations.Add(topologyWatcherOpRemoveTablet, 1) + } + } + tw.tablets = newTablets + if !tw.firstLoadDone { + tw.firstLoadDone = true + close(tw.firstLoadChan) + } + + // iterate through the tablets in a stable order and compute a + // checksum of the tablet map + sort.Strings(tabletAliasStrs) + var buf bytes.Buffer + for _, alias := range tabletAliasStrs { + _, ok := tw.tablets[alias] + if ok { + buf.WriteString(alias) + } + } + tw.topoChecksum = crc32.ChecksumIEEE(buf.Bytes()) + tw.lastRefresh = time.Now() + + tw.mu.Unlock() + +} + +func (tw *TopologyWatcher) getAliasByCell(cell string) string { + tw.mu.Lock() + defer tw.mu.Unlock() + + if alias, ok := tw.cellAliases[cell]; ok { + return alias + } + + alias := topo.GetAliasByCell(context.Background(), tw.topoServer, cell) + tw.cellAliases[cell] = alias + + return alias +} + +func (tw *TopologyWatcher) isTabletInCell(tablet *topodata.Tablet) bool { + if tablet.Type == topodata.TabletType_MASTER { + return true + } + if tablet.Alias.Cell == tw.cell { + return true + } + if tw.getAliasByCell(tablet.Alias.Cell) == tw.getAliasByCell(tw.cell) { + return true + } + return false +} + // RefreshLag returns the time since the last refresh func (tw *TopologyWatcher) RefreshLag() time.Duration { tw.mu.Lock() @@ -153,7 +309,7 @@ func (tw *TopologyWatcher) TopoChecksum() uint32 { // to be applied as an additional filter on the list of tablets returned by its getTablets function type TabletFilter interface { // IsIncluded returns whether tablet is included in this filter - IsIncluded(tablet *topodatapb.Tablet) bool + IsIncluded(tablet *topodata.Tablet) bool } // FilterByShard is a filter that filters tablets by @@ -168,7 +324,7 @@ type FilterByShard struct { type filterShard struct { keyspace string shard string - keyRange *topodatapb.KeyRange // only set if shard is also a KeyRange + keyRange *topodata.KeyRange // only set if shard is also a KeyRange } // NewFilterByShard creates a new FilterByShard on top of an existing @@ -214,7 +370,7 @@ func NewFilterByShard(filters []string) (*FilterByShard, error) { // IsIncluded returns true iff the tablet's keyspace and shard should be // forwarded to the underlying LegacyTabletRecorder. -func (fbs *FilterByShard) IsIncluded(tablet *topodatapb.Tablet) bool { +func (fbs *FilterByShard) IsIncluded(tablet *topodata.Tablet) bool { canonical, kr, err := topo.ValidateShardName(tablet.Shard) if err != nil { log.Errorf("Error parsing shard name %v, will ignore tablet: %v", tablet.Shard, err) @@ -243,7 +399,7 @@ type FilterByKeyspace struct { // NewFilterByKeyspace creates a new FilterByKeyspace. // Each filter is a keyspace entry. All tablets that match // a keyspace will be forwarded to the underlying LegacyTabletRecorder. -func NewFilterByKeyspace(cell string, selectedKeyspaces []string) *FilterByKeyspace { +func NewFilterByKeyspace(selectedKeyspaces []string) *FilterByKeyspace { m := make(map[string]bool) for _, keyspace := range selectedKeyspaces { m[keyspace] = true @@ -256,7 +412,7 @@ func NewFilterByKeyspace(cell string, selectedKeyspaces []string) *FilterByKeysp // IsIncluded returns true if the tablet's keyspace should be // forwarded to the underlying LegacyTabletRecorder. -func (fbk *FilterByKeyspace) IsIncluded(tablet *topodatapb.Tablet) bool { +func (fbk *FilterByKeyspace) IsIncluded(tablet *topodata.Tablet) bool { _, exist := fbk.keyspaces[tablet.Keyspace] return exist } diff --git a/go/vt/discovery/topology_watcher_test.go b/go/vt/discovery/topology_watcher_test.go new file mode 100644 index 00000000000..86dd652c7cf --- /dev/null +++ b/go/vt/discovery/topology_watcher_test.go @@ -0,0 +1,460 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "math/rand" + "testing" + "time" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + "vitess.io/vitess/go/vt/logutil" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/memorytopo" +) + +func checkOpCounts(t *testing.T, prevCounts, deltas map[string]int64) map[string]int64 { + t.Helper() + newCounts := topologyWatcherOperations.Counts() + for key, prevVal := range prevCounts { + delta, ok := deltas[key] + if !ok { + delta = 0 + } + newVal, ok := newCounts[key] + if !ok { + newVal = 0 + } + + if newVal != prevVal+delta { + t.Errorf("expected %v to increase by %v, got %v -> %v", key, delta, prevVal, newVal) + } + } + return newCounts +} + +func checkChecksum(t *testing.T, tw *TopologyWatcher, want uint32) { + t.Helper() + got := tw.TopoChecksum() + if want != got { + t.Errorf("want checksum %v got %v", want, got) + } +} + +func TestCellTabletsWatcher(t *testing.T) { + checkWatcher(t, true) +} + +func TestCellTabletsWatcherNoRefreshKnown(t *testing.T) { + checkWatcher(t, false) +} + +func checkWatcher(t *testing.T, refreshKnownTablets bool) { + ts := memorytopo.NewServer("aa") + fhc := NewFakeHealthCheck() + logger := logutil.NewMemoryLogger() + topologyWatcherOperations.ZeroAll() + counts := topologyWatcherOperations.Counts() + tw := NewCellTabletsWatcher(context.Background(), ts, fhc, nil, "aa", 10*time.Minute, refreshKnownTablets, 5) + + counts = checkOpCounts(t, counts, map[string]int64{}) + checkChecksum(t, tw, 0) + + // Add a tablet to the topology. + tablet := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "aa", + Uid: 0, + }, + Hostname: "host1", + PortMap: map[string]int32{ + "vt": 123, + }, + Keyspace: "keyspace", + Shard: "shard", + } + if err := ts.CreateTablet(context.Background(), tablet); err != nil { + t.Fatalf("CreateTablet failed: %v", err) + } + tw.loadTablets() + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "AddTablet": 1}) + checkChecksum(t, tw, 3238442862) + + // Check the tablet is returned by GetAllTablets(). + allTablets := fhc.GetAllTablets() + key := TabletToMapKey(tablet) + if _, ok := allTablets[key]; !ok || len(allTablets) != 1 || !proto.Equal(allTablets[key], tablet) { + t.Errorf("fhc.GetAllTablets() = %+v; want %+v", allTablets, tablet) + } + + // Add a second tablet to the topology. + tablet2 := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "aa", + Uid: 2, + }, + Hostname: "host2", + PortMap: map[string]int32{ + "vt": 789, + }, + Keyspace: "keyspace", + Shard: "shard", + } + if err := ts.CreateTablet(context.Background(), tablet2); err != nil { + t.Fatalf("CreateTablet failed: %v", err) + } + tw.loadTablets() + + // If RefreshKnownTablets is disabled, only the new tablet is read + // from the topo + if refreshKnownTablets { + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "AddTablet": 1}) + } else { + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "AddTablet": 1}) + } + checkChecksum(t, tw, 2762153755) + + // Check the new tablet is returned by GetAllTablets(). + allTablets = fhc.GetAllTablets() + key = TabletToMapKey(tablet2) + if _, ok := allTablets[key]; !ok || len(allTablets) != 2 || !proto.Equal(allTablets[key], tablet2) { + t.Errorf("fhc.GetAllTablets() = %+v; want %+v", allTablets, tablet2) + } + + // Load the tablets again to show that when RefreshKnownTablets is disabled, + // only the list is read from the topo and the checksum doesn't change + tw.loadTablets() + if refreshKnownTablets { + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2}) + } else { + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1}) + } + checkChecksum(t, tw, 2762153755) + + // same tablet, different port, should update (previous + // one should go away, new one be added) + // + // if RefreshKnownTablets is disabled, this case is *not* + // detected and the tablet remains in the topo using the + // old key + origTablet := proto.Clone(tablet).(*topodatapb.Tablet) + origKey := TabletToMapKey(tablet) + tablet.PortMap["vt"] = 456 + if _, err := ts.UpdateTabletFields(context.Background(), tablet.Alias, func(t *topodatapb.Tablet) error { + t.PortMap["vt"] = 456 + return nil + }); err != nil { + t.Fatalf("UpdateTabletFields failed: %v", err) + } + tw.loadTablets() + allTablets = fhc.GetAllTablets() + key = TabletToMapKey(tablet) + + if refreshKnownTablets { + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 1}) + + if _, ok := allTablets[key]; !ok || len(allTablets) != 2 || !proto.Equal(allTablets[key], tablet) { + t.Errorf("fhc.GetAllTablets() = %+v; want %+v", allTablets, tablet) + } + if _, ok := allTablets[origKey]; ok { + t.Errorf("fhc.GetAllTablets() = %+v; don't want %v", allTablets, origKey) + } + checkChecksum(t, tw, 2762153755) + } else { + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1}) + + if _, ok := allTablets[origKey]; !ok || len(allTablets) != 2 || !proto.Equal(allTablets[origKey], origTablet) { + t.Errorf("fhc.GetAllTablets() = %+v; want %+v", allTablets, origTablet) + } + if _, ok := allTablets[key]; ok { + t.Errorf("fhc.GetAllTablets() = %+v; don't want %v", allTablets, key) + } + checkChecksum(t, tw, 2762153755) + } + + // Both tablets restart on different hosts. + // tablet2 happens to land on the host:port that tablet 1 used to be on. + // This can only be tested when we refresh known tablets. + if refreshKnownTablets { + origTablet := *tablet + origTablet2 := *tablet2 + + if _, err := ts.UpdateTabletFields(context.Background(), tablet2.Alias, func(t *topodatapb.Tablet) error { + t.Hostname = tablet.Hostname + t.PortMap = tablet.PortMap + tablet2 = t + return nil + }); err != nil { + t.Fatalf("UpdateTabletFields failed: %v", err) + } + if _, err := ts.UpdateTabletFields(context.Background(), tablet.Alias, func(t *topodatapb.Tablet) error { + t.Hostname = "host3" + tablet = t + return nil + }); err != nil { + t.Fatalf("UpdateTabletFields failed: %v", err) + } + tw.loadTablets() + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 2}) + allTablets = fhc.GetAllTablets() + key2 := TabletToMapKey(tablet2) + if _, ok := allTablets[key2]; !ok { + t.Fatalf("tablet was lost because it's reusing an address recently used by another tablet: %v", key2) + } + + // Change tablets back to avoid altering later tests. + if _, err := ts.UpdateTabletFields(context.Background(), tablet2.Alias, func(t *topodatapb.Tablet) error { + t.Hostname = origTablet2.Hostname + t.PortMap = origTablet2.PortMap + tablet2 = t + return nil + }); err != nil { + t.Fatalf("UpdateTabletFields failed: %v", err) + } + if _, err := ts.UpdateTabletFields(context.Background(), tablet.Alias, func(t *topodatapb.Tablet) error { + t.Hostname = origTablet.Hostname + tablet = t + return nil + }); err != nil { + t.Fatalf("UpdateTabletFields failed: %v", err) + } + tw.loadTablets() + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "ReplaceTablet": 2}) + } + + // Remove the tablet and check that it is detected as being gone. + if err := ts.DeleteTablet(context.Background(), tablet.Alias); err != nil { + t.Fatalf("DeleteTablet failed: %v", err) + } + if err := topo.FixShardReplication(context.Background(), ts, logger, "aa", "keyspace", "shard"); err != nil { + t.Fatalf("FixShardReplication failed: %v", err) + } + tw.loadTablets() + if refreshKnownTablets { + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "RemoveTablet": 1}) + } else { + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "RemoveTablet": 1}) + } + checkChecksum(t, tw, 789108290) + + allTablets = fhc.GetAllTablets() + key = TabletToMapKey(tablet) + if _, ok := allTablets[key]; ok || len(allTablets) != 1 { + t.Errorf("fhc.GetAllTablets() = %+v; don't want %v", allTablets, key) + } + key = TabletToMapKey(tablet2) + if _, ok := allTablets[key]; !ok || len(allTablets) != 1 || !proto.Equal(allTablets[key], tablet2) { + t.Errorf("fhc.GetAllTablets() = %+v; want %+v", allTablets, tablet2) + } + + // Remove the other and check that it is detected as being gone. + if err := ts.DeleteTablet(context.Background(), tablet2.Alias); err != nil { + t.Fatalf("DeleteTablet failed: %v", err) + } + if err := topo.FixShardReplication(context.Background(), ts, logger, "aa", "keyspace", "shard"); err != nil { + t.Fatalf("FixShardReplication failed: %v", err) + } + tw.loadTablets() + checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 0, "RemoveTablet": 1}) + checkChecksum(t, tw, 0) + + allTablets = fhc.GetAllTablets() + key = TabletToMapKey(tablet) + if _, ok := allTablets[key]; ok || len(allTablets) != 0 { + t.Errorf("fhc.GetAllTablets() = %+v; don't want %v", allTablets, key) + } + key = TabletToMapKey(tablet2) + if _, ok := allTablets[key]; ok || len(allTablets) != 0 { + t.Errorf("fhc.GetAllTablets() = %+v; don't want %v", allTablets, key) + } + + tw.Stop() +} + +func TestFilterByShard(t *testing.T) { + testcases := []struct { + filters []string + keyspace string + shard string + included bool + }{ + // un-sharded keyspaces + { + filters: []string{"ks1|0"}, + keyspace: "ks1", + shard: "0", + included: true, + }, + { + filters: []string{"ks1|0"}, + keyspace: "ks2", + shard: "0", + included: false, + }, + // custom sharding, different shard + { + filters: []string{"ks1|0"}, + keyspace: "ks1", + shard: "1", + included: false, + }, + // keyrange based sharding + { + filters: []string{"ks1|-80"}, + keyspace: "ks1", + shard: "0", + included: false, + }, + { + filters: []string{"ks1|-80"}, + keyspace: "ks1", + shard: "-40", + included: true, + }, + { + filters: []string{"ks1|-80"}, + keyspace: "ks1", + shard: "-80", + included: true, + }, + { + filters: []string{"ks1|-80"}, + keyspace: "ks1", + shard: "80-", + included: false, + }, + { + filters: []string{"ks1|-80"}, + keyspace: "ks1", + shard: "c0-", + included: false, + }, + } + + for _, tc := range testcases { + fbs, err := NewFilterByShard(tc.filters) + if err != nil { + t.Errorf("cannot create FilterByShard for filters %v: %v", tc.filters, err) + } + + tablet := &topodatapb.Tablet{ + Keyspace: tc.keyspace, + Shard: tc.shard, + } + + got := fbs.IsIncluded(tablet) + if got != tc.included { + t.Errorf("isIncluded(%v,%v) for filters %v returned %v but expected %v", tc.keyspace, tc.shard, tc.filters, got, tc.included) + } + } +} + +var ( + testFilterByKeyspace = []struct { + keyspace string + expected bool + }{ + {"ks1", true}, + {"ks2", true}, + {"ks3", false}, + {"ks4", true}, + {"ks5", true}, + {"ks6", false}, + {"ks7", false}, + } + testKeyspacesToWatch = []string{"ks1", "ks2", "ks4", "ks5"} + testCell = "testCell" + testShard = "testShard" + testHostName = "testHostName" +) + +func TestFilterByKeyspace(t *testing.T) { + hc := NewFakeHealthCheck() + f := NewFilterByKeyspace(testKeyspacesToWatch) + ts := memorytopo.NewServer(testCell) + tw := NewCellTabletsWatcher(context.Background(), ts, hc, f, testCell, 10*time.Minute, true, 5) + + for _, test := range testFilterByKeyspace { + // Add a new tablet to the topology. + port := rand.Int31n(1000) + tablet := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: testCell, + Uid: rand.Uint32(), + }, + Hostname: testHostName, + PortMap: map[string]int32{ + "vt": port, + }, + Keyspace: test.keyspace, + Shard: testShard, + } + + got := f.IsIncluded(tablet) + if got != test.expected { + t.Errorf("isIncluded(%v) for keyspace %v returned %v but expected %v", test.keyspace, test.keyspace, got, test.expected) + } + + if err := ts.CreateTablet(context.Background(), tablet); err != nil { + t.Errorf("CreateTablet failed: %v", err) + } + + tw.loadTablets() + key := TabletToMapKey(tablet) + allTablets := hc.GetAllTablets() + + if _, ok := allTablets[key]; ok != test.expected && proto.Equal(allTablets[key], tablet) != test.expected { + t.Errorf("Error adding tablet - got %v; want %v", ok, test.expected) + } + + // Replace the tablet we added above + tabletReplacement := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: testCell, + Uid: rand.Uint32(), + }, + Hostname: testHostName, + PortMap: map[string]int32{ + "vt": port, + }, + Keyspace: test.keyspace, + Shard: testShard, + } + got = f.IsIncluded(tabletReplacement) + if got != test.expected { + t.Errorf("isIncluded(%v) for keyspace %v returned %v but expected %v", test.keyspace, test.keyspace, got, test.expected) + } + if err := ts.CreateTablet(context.Background(), tabletReplacement); err != nil { + t.Errorf("CreateTablet failed: %v", err) + } + + tw.loadTablets() + key = TabletToMapKey(tabletReplacement) + allTablets = hc.GetAllTablets() + + if _, ok := allTablets[key]; ok != test.expected && proto.Equal(allTablets[key], tabletReplacement) != test.expected { + t.Errorf("Error replacing tablet - got %v; want %v", ok, test.expected) + } + + // Delete the tablet + if err := ts.DeleteTablet(context.Background(), tabletReplacement.Alias); err != nil { + t.Fatalf("DeleteTablet failed: %v", err) + } + } +} From 8946b0690c13a8801aa499af3b59cf3acdace3c3 Mon Sep 17 00:00:00 2001 From: deepthi Date: Fri, 8 May 2020 15:09:16 -0700 Subject: [PATCH 23/39] healthcheck: fix stats and healthy, endtoend test Signed-off-by: deepthi --- .../endtoend/vtgate/healthcheck/main_test.go | 100 +++++++++++++++ .../vtgate/healthcheck/vtgate_test.go | 121 ++++++++++++++++++ go/vt/discovery/healthcheck.go | 44 +++---- go/vt/discovery/tablet_health.go | 4 - 4 files changed, 236 insertions(+), 33 deletions(-) create mode 100644 go/test/endtoend/vtgate/healthcheck/main_test.go create mode 100644 go/test/endtoend/vtgate/healthcheck/vtgate_test.go diff --git a/go/test/endtoend/vtgate/healthcheck/main_test.go b/go/test/endtoend/vtgate/healthcheck/main_test.go new file mode 100644 index 00000000000..c3431c41a4b --- /dev/null +++ b/go/test/endtoend/vtgate/healthcheck/main_test.go @@ -0,0 +1,100 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package healthcheck + +import ( + "flag" + "os" + "testing" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + keyspaceName = "commerce" + cell = "zone1" + sqlSchema = `create table product( + sku varbinary(128), + description varbinary(128), + price bigint, + primary key(sku) + ) ENGINE=InnoDB; + create table customer( + id bigint not null auto_increment, + email varchar(128), + primary key(id) + ) ENGINE=InnoDB; + create table corder( + order_id bigint not null auto_increment, + customer_id bigint, + sku varbinary(128), + price bigint, + primary key(order_id) + ) ENGINE=InnoDB;` + + vSchema = `{ + "tables": { + "product": {}, + "customer": {}, + "corder": {} + } + }` +) + +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(cell, "localhost") + clusterInstance.VtGateExtraArgs = []string{"-gateway_implementation", "tabletgateway"} + clusterInstance.VtTabletExtraArgs = []string{"-health_check_interval", "1s"} + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + return 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + SchemaSQL: sqlSchema, + VSchema: vSchema, + } + err = clusterInstance.StartUnshardedKeyspace(*keyspace, 1, true) + if err != nil { + return 1 + } + + // Start vtgate + err = clusterInstance.StartVtgate() + if err != nil { + return 1 + } + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + return m.Run() + }() + os.Exit(exitCode) +} diff --git a/go/test/endtoend/vtgate/healthcheck/vtgate_test.go b/go/test/endtoend/vtgate/healthcheck/vtgate_test.go new file mode 100644 index 00000000000..9b39b37d991 --- /dev/null +++ b/go/test/endtoend/vtgate/healthcheck/vtgate_test.go @@ -0,0 +1,121 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This tests select/insert using the unshared keyspace added in main_test +*/ +package healthcheck + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "reflect" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +func TestVtgateProcess(t *testing.T) { + defer cluster.PanicHandler(t) + // Healthcheck interval on tablet is set to 1s, so sleep for 2s + time.Sleep(2 * time.Second) + verifyVtgateVariables(t, clusterInstance.VtgateProcess.VerifyURL) + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.Nil(t, err) + defer conn.Close() + + exec(t, conn, "insert into customer(id, email) values(1,'email1')") + + qr := exec(t, conn, "select id, email from customer") + if got, want := fmt.Sprintf("%v", qr.Rows), `[[INT64(1) VARCHAR("email1")]]`; got != want { + t.Errorf("select:\n%v want\n%v", got, want) + } +} + +func verifyVtgateVariables(t *testing.T, url string) { + resp, _ := http.Get(url) + if resp != nil && resp.StatusCode == 200 { + resultMap := make(map[string]interface{}) + respByte, _ := ioutil.ReadAll(resp.Body) + err := json.Unmarshal(respByte, &resultMap) + require.Nil(t, err) + if resultMap["VtgateVSchemaCounts"] == nil { + t.Error("Vschema count should be present in variables") + } + vschemaCountMap := getMapFromJSON(resultMap, "VtgateVSchemaCounts") + if _, present := vschemaCountMap["Reload"]; !present { + t.Error("Reload count should be present in vschemacount") + } else if object := reflect.ValueOf(vschemaCountMap["Reload"]); object.NumField() <= 0 { + t.Error("Reload count should be greater than 0") + } + if _, present := vschemaCountMap["WatchError"]; present { + t.Error("There should not be any WatchError in VschemaCount") + } + if _, present := vschemaCountMap["Parsing"]; present { + t.Error("There should not be any Parsing in VschemaCount") + } + + if resultMap["HealthcheckConnections"] == nil { + t.Error("HealthcheckConnections count should be present in variables") + } + + healthCheckConnection := getMapFromJSON(resultMap, "HealthcheckConnections") + if len(healthCheckConnection) <= 0 { + t.Error("Atleast one healthy tablet needs to be present") + } + if !isMasterTabletPresent(healthCheckConnection) { + t.Error("Atleast one master tablet needs to be present") + } + } else { + t.Error("Vtgate api url response not found") + } +} + +func getMapFromJSON(JSON map[string]interface{}, key string) map[string]interface{} { + result := make(map[string]interface{}) + object := reflect.ValueOf(JSON[key]) + if object.Kind() == reflect.Map { + for _, key := range object.MapKeys() { + value := object.MapIndex(key) + result[key.String()] = value + } + } + return result +} + +func isMasterTabletPresent(tablets map[string]interface{}) bool { + for key := range tablets { + if strings.Contains(key, "master") { + return true + } + } + return false +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.Nil(t, err) + return qr +} diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 1ef7c05a136..7462f17fe93 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -201,7 +201,6 @@ type HealthCheckImpl struct { // has to be kept in sync with healthByAlias healthData map[string]map[string]*tabletHealthCheck // another map keyed by keyspace.shard.tabletType, this one containing a sorted list of tabletHealthCheck - // TODO(deepthi): replace with TabletHealth healthy map[string][]*tabletHealthCheck // connsWG keeps track of all launched Go routines that monitor tablet connections. connsWG sync.WaitGroup @@ -327,7 +326,7 @@ func (hc *HealthCheckImpl) RegisterStats() { ) stats.NewGaugesFuncWithMultiLabels( - "TabletHealthections", + "HealthcheckConnections", "the number of healthcheck connections registered", []string{"Keyspace", "ShardName", "TabletType"}, hc.servingConnStats) @@ -358,13 +357,15 @@ func (hc *HealthCheckImpl) servingConnStats() map[string]int64 { res := make(map[string]int64) hc.mu.Lock() defer hc.mu.Unlock() - for key, ths := range hc.healthData { - for _, th := range ths { - if !th.Serving || th.LastError != nil { - continue - } - res[key]++ + for _, th := range hc.healthByAlias { + th.mu.Lock() + if !th.Serving || th.LastError != nil { + th.mu.Unlock() + continue } + key := fmt.Sprintf("%s.%s.%s", th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)) + th.mu.Unlock() + res[key]++ } return res } @@ -669,28 +670,11 @@ func (hc *HealthCheckImpl) topologyWatcherChecksum() int64 { // synchronization func (hc *HealthCheckImpl) GetHealthyTabletStats(target *query.Target) []*TabletHealth { var result []*TabletHealth - // we check all tablet types in all cells because of cellAliases - key := hc.keyFromTarget(target) - ths, ok := hc.healthData[key] - if !ok { - log.Warningf("Healthcheck has no tablet health for target: %v", key) - return result - } - if target.TabletType == topodata.TabletType_MASTER && len(ths) > 1 { - log.Warningf("Can only have one master, program bug: %v", ths) - return result - } - for _, th := range hc.healthByAlias { - if th.Tablet.Type == topodata.TabletType_MASTER { - result = append(result, th.SimpleCopy()) - return result - } - if th.isHealthy() { - result = append(result, th.SimpleCopy()) - } + hc.mu.Lock() + defer hc.mu.Unlock() + for _, thc := range hc.healthy[hc.keyFromTarget(target)] { + result = append(result, thc.SimpleCopy()) } - // healthy list needs to be sorted using replication lag algorithm - // so we might want to maintain it and update it instead of computing it here return result } @@ -700,6 +684,8 @@ func (hc *HealthCheckImpl) GetHealthyTabletStats(target *query.Target) []*Tablet // the most recent tablet of type master. func (hc *HealthCheckImpl) getTabletStats(target *query.Target) []*TabletHealth { var result []*TabletHealth + hc.mu.Lock() + defer hc.mu.Unlock() ths := hc.healthData[hc.keyFromTarget(target)] for _, th := range ths { result = append(result, th.SimpleCopy()) diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 0050d782dce..0a72f4a768b 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -168,10 +168,6 @@ func (th *tabletHealthCheck) deleteConnLocked() { th.cancelFunc() } -func (th *tabletHealthCheck) isHealthy() bool { - return th.Serving && th.LastError == nil && th.Stats != nil && !IsReplicationLagVeryHigh(th) -} - // setServingState sets the tablet state to the given value. // // If the state changes, it logs the change so that failures From 2b95b2554084e9a2eac0733beca76db066f9a7e7 Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 11 May 2020 19:43:59 -0700 Subject: [PATCH 24/39] healthcheck: endtoend test for scatter_conn and buffer Signed-off-by: deepthi --- .../vtgate/healthcheck/buffer/buffer_test.go | 394 ++++++++++++++++++ .../vtgate/healthcheck/vtgate_test.go | 62 ++- go/vt/discovery/tablet_health.go | 33 +- go/vt/vtgate/executor.go | 2 +- 4 files changed, 439 insertions(+), 52 deletions(-) create mode 100644 go/test/endtoend/vtgate/healthcheck/buffer/buffer_test.go diff --git a/go/test/endtoend/vtgate/healthcheck/buffer/buffer_test.go b/go/test/endtoend/vtgate/healthcheck/buffer/buffer_test.go new file mode 100644 index 00000000000..1cf1e768c0c --- /dev/null +++ b/go/test/endtoend/vtgate/healthcheck/buffer/buffer_test.go @@ -0,0 +1,394 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Test the vtgate master buffer. + +During a master failover, vtgate should automatically buffer (stall) requests +for a configured time and retry them after the failover is over. + +The test reproduces such a scenario as follows: +- run two threads, the first thread continuously executes a critical read and the second executes a write (UPDATE) +- vtctl PlannedReparentShard runs a master failover +- both threads should not see any error during the failover +*/ + +package buffer + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "os" + "reflect" + "strconv" + "strings" + "sync" + "testing" + "time" + + "vitess.io/vitess/go/vt/log" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + keyspaceUnshardedName = "ks1" + cell = "zone1" + hostname = "localhost" + sqlSchema = ` + create table buffer( + id BIGINT NOT NULL, + msg VARCHAR(64) NOT NULL, + PRIMARY KEY (id) + ) Engine=InnoDB;` + wg = &sync.WaitGroup{} +) + +const ( + criticalReadRowID = 1 + updateRowID = 2 + demoteMasterQuery = "SET GLOBAL read_only = ON;FLUSH TABLES WITH READ LOCK;UNLOCK TABLES;" + disableSemiSyncMasterQuery = "SET GLOBAL rpl_semi_sync_master_enabled = 0" + enableSemiSyncMasterQuery = "SET GLOBAL rpl_semi_sync_master_enabled = 1" + promoteSlaveQuery = "STOP SLAVE;RESET SLAVE ALL;SET GLOBAL read_only = OFF;" +) + +//threadParams is set of params passed into read and write threads +type threadParams struct { + writable bool + quit bool + rpcs int // Number of queries successfully executed. + errors int // Number of failed queries. + waitForNotification chan bool // Channel used to notify the main thread that this thread executed + notifyLock sync.Mutex // notifyLock guards the two fields notifyAfterNSuccessfulRpcs/rpcsSoFar. + notifyAfterNSuccessfulRpcs int // If 0, notifications are disabled + rpcsSoFar int // Number of RPCs at the time a notification was requested + i int // + commitErrors int + executeFunction func(c *threadParams, conn *mysql.Conn) error // Implement the method for read/update. +} + +// Thread which constantly executes a query on vtgate. +func (c *threadParams) threadRun() { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + if err != nil { + log.Errorf("error connecting to mysql with params %v: %v", vtParams, err) + } + defer conn.Close() + for !c.quit { + err = c.executeFunction(c, conn) + if err != nil { + c.errors++ + log.Errorf("error executing function %v: %v", c.executeFunction, err) + } + c.rpcs++ + // If notifications are requested, check if we already executed the + // required number of successful RPCs. + // Use >= instead of == because we can miss the exact point due to + // slow thread scheduling. + c.notifyLock.Lock() + if c.notifyAfterNSuccessfulRpcs != 0 && c.rpcs >= (c.notifyAfterNSuccessfulRpcs+c.rpcsSoFar) { + c.waitForNotification <- true + c.notifyAfterNSuccessfulRpcs = 0 + } + c.notifyLock.Unlock() + // Wait 10ms seconds between two attempts. + time.Sleep(10 * time.Millisecond) + } + wg.Done() +} + +func (c *threadParams) setNotifyAfterNSuccessfulRpcs(n int) { + c.notifyLock.Lock() + c.notifyAfterNSuccessfulRpcs = n + c.rpcsSoFar = c.rpcs + c.notifyLock.Unlock() +} + +func (c *threadParams) stop() { + c.quit = true +} + +func readExecute(c *threadParams, conn *mysql.Conn) error { + _, err := conn.ExecuteFetch(fmt.Sprintf("SELECT * FROM buffer WHERE id = %d", criticalReadRowID), 1000, true) + return err +} + +func updateExecute(c *threadParams, conn *mysql.Conn) error { + attempt := c.i + // Value used in next UPDATE query. Increased after every query. + c.i++ + conn.ExecuteFetch("begin", 1000, true) + + result, err := conn.ExecuteFetch(fmt.Sprintf("UPDATE buffer SET msg='update %d' WHERE id = %d", attempt, updateRowID), 1000, true) + + // Sleep between [0, 1] seconds to prolong the time the transaction is in + // flight. This is more realistic because applications are going to keep + // their transactions open for longer as well. + time.Sleep(time.Duration(rand.Int31n(1000)) * time.Millisecond) + + if err == nil { + log.Infof("update attempt #%d affected %v rows", attempt, result.RowsAffected) + _, err = conn.ExecuteFetch("commit", 1000, true) + if err != nil { + _, errRollback := conn.ExecuteFetch("rollback", 1000, true) + if errRollback != nil { + log.Errorf("Error in rollback: %v", errRollback) + } + c.commitErrors++ + if c.commitErrors > 1 { + return err + } + log.Errorf("UPDATE %d failed during ROLLBACK. This is okay once because we do not support buffering it. err: %v", attempt, err) + } + } + if err != nil { + _, errRollback := conn.ExecuteFetch("rollback", 1000, true) + if errRollback != nil { + log.Errorf("Error in rollback: %v", errRollback) + } + c.commitErrors++ + if c.commitErrors > 1 { + return err + } + log.Errorf("UPDATE %d failed during COMMIT with err: %v.This is okay once because we do not support buffering it.", attempt, err) + } + return nil +} + +func createCluster() (*cluster.LocalProcessCluster, int) { + clusterInstance = cluster.NewCluster(cell, hostname) + + // Start topo server + if err := clusterInstance.StartTopo(); err != nil { + return nil, 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceUnshardedName, + SchemaSQL: sqlSchema, + } + clusterInstance.VtTabletExtraArgs = []string{"-health_check_interval", "1s"} + + if err := clusterInstance.StartUnshardedKeyspace(*keyspace, 1, false); err != nil { + return nil, 1 + } + + clusterInstance.VtGateExtraArgs = []string{ + "-enable_buffer", + // Long timeout in case failover is slow. + "-buffer_window", "10m", + "-buffer_max_failover_duration", "10m", + "-buffer_min_time_between_failovers", "20m", + "-gateway_implementation", "tabletgateway"} + + // Start vtgate + if err := clusterInstance.StartVtgate(); err != nil { + return nil, 1 + } + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + rand.Seed(time.Now().UnixNano()) + return clusterInstance, 0 +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.Nil(t, err) + return qr +} + +func TestBufferInternalReparenting(t *testing.T) { + testBufferBase(t, false) +} + +func TestBufferExternalReparenting(t *testing.T) { + testBufferBase(t, true) +} + +func testBufferBase(t *testing.T, isExternalParent bool) { + defer cluster.PanicHandler(t) + clusterInstance, exitCode := createCluster() + if exitCode != 0 { + os.Exit(exitCode) + } + // Healthcheck interval on tablet is set to 1s, so sleep for 2s + time.Sleep(2 * time.Second) + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.Nil(t, err) + defer conn.Close() + + // Insert two rows for the later threads (critical read, update). + exec(t, conn, fmt.Sprintf("INSERT INTO buffer (id, msg) VALUES (%d, %s)", criticalReadRowID, "'critical read'")) + exec(t, conn, fmt.Sprintf("INSERT INTO buffer (id, msg) VALUES (%d, %s)", updateRowID, "'update'")) + + //Start both threads. + readThreadInstance := &threadParams{writable: false, quit: false, rpcs: 0, errors: 0, notifyAfterNSuccessfulRpcs: 0, rpcsSoFar: 0, executeFunction: readExecute, waitForNotification: make(chan bool)} + wg.Add(1) + go readThreadInstance.threadRun() + updateThreadInstance := &threadParams{writable: false, quit: false, rpcs: 0, errors: 0, notifyAfterNSuccessfulRpcs: 0, rpcsSoFar: 0, executeFunction: updateExecute, i: 1, commitErrors: 0, waitForNotification: make(chan bool)} + wg.Add(1) + go updateThreadInstance.threadRun() + + // Verify they got at least 2 RPCs through. + readThreadInstance.setNotifyAfterNSuccessfulRpcs(2) + updateThreadInstance.setNotifyAfterNSuccessfulRpcs(2) + + <-readThreadInstance.waitForNotification + <-updateThreadInstance.waitForNotification + + // Execute the failover. + readThreadInstance.setNotifyAfterNSuccessfulRpcs(10) + updateThreadInstance.setNotifyAfterNSuccessfulRpcs(10) + + if isExternalParent { + externalReparenting(ctx, t, clusterInstance) + } else { + //reparent call + clusterInstance.VtctlclientProcess.ExecuteCommand("PlannedReparentShard", "-keyspace_shard", + fmt.Sprintf("%s/%s", keyspaceUnshardedName, "0"), + "-new_master", clusterInstance.Keyspaces[0].Shards[0].Vttablets[1].Alias) + } + + <-readThreadInstance.waitForNotification + <-updateThreadInstance.waitForNotification + + // Stop threads + readThreadInstance.stop() + updateThreadInstance.stop() + + // Both threads must not see any error + assert.Equal(t, 0, readThreadInstance.errors) + assert.Equal(t, 0, updateThreadInstance.errors) + + //At least one thread should have been buffered. + //This may fail if a failover is too fast. Add retries then. + resp, err := http.Get(clusterInstance.VtgateProcess.VerifyURL) + require.Nil(t, err) + label := fmt.Sprintf("%s.%s", keyspaceUnshardedName, "0") + inFlightMax := 0 + masterPromotedCount := 0 + durationMs := 0 + bufferingStops := 0 + if resp.StatusCode == 200 { + resultMap := make(map[string]interface{}) + respByte, _ := ioutil.ReadAll(resp.Body) + err := json.Unmarshal(respByte, &resultMap) + if err != nil { + panic(err) + } + inFlightMax = getVarFromVtgate(t, label, "BufferLastRequestsInFlightMax", resultMap) + masterPromotedCount = getVarFromVtgate(t, label, "HealthcheckMasterPromoted", resultMap) + durationMs = getVarFromVtgate(t, label, "BufferFailoverDurationSumMs", resultMap) + bufferingStops = getVarFromVtgate(t, "NewMasterSeen", "BufferStops", resultMap) + } + if inFlightMax == 0 { + // Missed buffering is okay when we observed the failover during the + // COMMIT (which cannot trigger the buffering). + assert.Greater(t, updateThreadInstance.commitErrors, 0, "No buffering took place and the update thread saw no error during COMMIT. But one of it must happen.") + } else { + assert.Greater(t, inFlightMax, 0) + } + + // There was a failover and the HealthCheck module must have seen it. + if masterPromotedCount > 0 { + assert.Greater(t, masterPromotedCount, 0) + } + + if durationMs > 0 { + // Number of buffering stops must be equal to the number of seen failovers. + assert.Equal(t, masterPromotedCount, bufferingStops) + } + wg.Wait() + clusterInstance.Teardown() +} + +func getVarFromVtgate(t *testing.T, label string, param string, resultMap map[string]interface{}) int { + paramVal := 0 + var err error + object := reflect.ValueOf(resultMap[param]) + if object.Kind() == reflect.Map { + for _, key := range object.MapKeys() { + if strings.Contains(key.String(), label) { + v := object.MapIndex(key) + s := fmt.Sprintf("%v", v.Interface()) + paramVal, err = strconv.Atoi(s) + require.Nil(t, err) + } + } + } + return paramVal +} + +func externalReparenting(ctx context.Context, t *testing.T, clusterInstance *cluster.LocalProcessCluster) { + start := time.Now() + + // Demote master Query + master := clusterInstance.Keyspaces[0].Shards[0].Vttablets[0] + replica := clusterInstance.Keyspaces[0].Shards[0].Vttablets[1] + oldMaster := master + newMaster := replica + master.VttabletProcess.QueryTablet(demoteMasterQuery, keyspaceUnshardedName, true) + if master.VttabletProcess.EnableSemiSync { + master.VttabletProcess.QueryTablet(disableSemiSyncMasterQuery, keyspaceUnshardedName, true) + } + + // Wait for replica to catch up to master. + cluster.WaitForReplicationPos(t, master, replica, "localhost", 60.0) + + duration := time.Since(start) + minUnavailabilityInS := 1.0 + if duration.Seconds() < minUnavailabilityInS { + w := minUnavailabilityInS - duration.Seconds() + log.Infof("Waiting for %.1f seconds because the failover was too fast (took only %.3f seconds)", w, duration.Seconds()) + time.Sleep(time.Duration(w) * time.Second) + } + + // Promote replica to new master. + replica.VttabletProcess.QueryTablet(promoteSlaveQuery, keyspaceUnshardedName, true) + + if replica.VttabletProcess.EnableSemiSync { + replica.VttabletProcess.QueryTablet(enableSemiSyncMasterQuery, keyspaceUnshardedName, true) + } + + // Configure old master to replicate from new master. + + _, gtID := cluster.GetMasterPosition(t, *newMaster, hostname) + + // Use 'localhost' as hostname because Travis CI worker hostnames + // are too long for MySQL replication. + changeMasterCommands := fmt.Sprintf("RESET SLAVE;SET GLOBAL gtid_slave_pos = '%s';CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%d ,MASTER_USER='vt_repl', MASTER_USE_GTID = slave_pos;START SLAVE;", gtID, "localhost", newMaster.MySQLPort) + oldMaster.VttabletProcess.QueryTablet(changeMasterCommands, keyspaceUnshardedName, true) + + // Notify the new vttablet master about the reparent. + clusterInstance.VtctlclientProcess.ExecuteCommand("TabletExternallyReparented", newMaster.Alias) +} diff --git a/go/test/endtoend/vtgate/healthcheck/vtgate_test.go b/go/test/endtoend/vtgate/healthcheck/vtgate_test.go index 9b39b37d991..cd67abe1978 100644 --- a/go/test/endtoend/vtgate/healthcheck/vtgate_test.go +++ b/go/test/endtoend/vtgate/healthcheck/vtgate_test.go @@ -28,6 +28,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" @@ -48,48 +49,33 @@ func TestVtgateProcess(t *testing.T) { exec(t, conn, "insert into customer(id, email) values(1,'email1')") qr := exec(t, conn, "select id, email from customer") - if got, want := fmt.Sprintf("%v", qr.Rows), `[[INT64(1) VARCHAR("email1")]]`; got != want { - t.Errorf("select:\n%v want\n%v", got, want) - } + assert.Equal(t, fmt.Sprintf("%v", qr.Rows), `[[INT64(1) VARCHAR("email1")]]`, "select returned wrong result") + qr = exec(t, conn, "show vitess_tablets") + assert.Equal(t, len(qr.Rows), 3, "wrong number of results from show") } func verifyVtgateVariables(t *testing.T, url string) { resp, _ := http.Get(url) - if resp != nil && resp.StatusCode == 200 { - resultMap := make(map[string]interface{}) - respByte, _ := ioutil.ReadAll(resp.Body) - err := json.Unmarshal(respByte, &resultMap) - require.Nil(t, err) - if resultMap["VtgateVSchemaCounts"] == nil { - t.Error("Vschema count should be present in variables") - } - vschemaCountMap := getMapFromJSON(resultMap, "VtgateVSchemaCounts") - if _, present := vschemaCountMap["Reload"]; !present { - t.Error("Reload count should be present in vschemacount") - } else if object := reflect.ValueOf(vschemaCountMap["Reload"]); object.NumField() <= 0 { - t.Error("Reload count should be greater than 0") - } - if _, present := vschemaCountMap["WatchError"]; present { - t.Error("There should not be any WatchError in VschemaCount") - } - if _, present := vschemaCountMap["Parsing"]; present { - t.Error("There should not be any Parsing in VschemaCount") - } - - if resultMap["HealthcheckConnections"] == nil { - t.Error("HealthcheckConnections count should be present in variables") - } - - healthCheckConnection := getMapFromJSON(resultMap, "HealthcheckConnections") - if len(healthCheckConnection) <= 0 { - t.Error("Atleast one healthy tablet needs to be present") - } - if !isMasterTabletPresent(healthCheckConnection) { - t.Error("Atleast one master tablet needs to be present") - } - } else { - t.Error("Vtgate api url response not found") - } + require.True(t, resp != nil && resp.StatusCode == 200, "Vtgate api url response not found") + resultMap := make(map[string]interface{}) + respByte, _ := ioutil.ReadAll(resp.Body) + err := json.Unmarshal(respByte, &resultMap) + require.Nil(t, err) + assert.True(t, resultMap["VtgateVSchemaCounts"] != nil, "Vschema count should be present in variables") + vschemaCountMap := getMapFromJSON(resultMap, "VtgateVSchemaCounts") + _, present := vschemaCountMap["Reload"] + assert.True(t, present, "Reload count should be present in vschemacount") + object := reflect.ValueOf(vschemaCountMap["Reload"]) + assert.True(t, object.NumField() > 0, "Reload count should be greater than 0") + _, present = vschemaCountMap["WatchError"] + assert.False(t, present, "There should not be any WatchError in VschemaCount") + _, present = vschemaCountMap["Parsing"] + assert.False(t, present, "There should not be any Parsing in VschemaCount") + + assert.True(t, resultMap["HealthcheckConnections"] != nil, "HealthcheckConnections count should be present in variables") + healthCheckConnection := getMapFromJSON(resultMap, "HealthcheckConnections") + assert.True(t, len(healthCheckConnection) > 0, "Atleast one healthy tablet needs to be present") + assert.True(t, isMasterTabletPresent(healthCheckConnection), "Atleast one master tablet needs to be present") } func getMapFromJSON(JSON map[string]interface{}, key string) map[string]interface{} { diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 0a72f4a768b..7964eff5aef 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -262,11 +262,14 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str } th.mu.Lock() - currentTablet := th.Tablet + currentTarget := th.Target // check whether this is a trivial update so as to update healthy map trivialNonMasterUpdate := th.LastError == nil && th.Serving && shr.RealtimeStats.HealthError == "" && shr.Serving && - currentTablet.Type != topodata.TabletType_MASTER && currentTablet.Type == shr.Target.TabletType - isMasterUpdate := currentTablet.Type == topodata.TabletType_MASTER && shr.Target.TabletType == topodata.TabletType_MASTER + currentTarget.TabletType != topodata.TabletType_MASTER && currentTarget.TabletType == shr.Target.TabletType + isMasterUpdate := shr.Target.TabletType == topodata.TabletType_MASTER + // Track how often a tablet gets promoted to master. It is used for + // comparing against the variables in go/vtgate/buffer/variables.go. + isMasterChange := currentTarget.TabletType != topodata.TabletType_MASTER && shr.Target.TabletType == topodata.TabletType_MASTER th.mu.Unlock() // hc.healthByAlias is authoritative, it should be updated @@ -277,12 +280,12 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str hc.mu.Unlock() hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) - if currentTablet.Type != shr.Target.TabletType || currentTablet.Keyspace != shr.Target.Keyspace || currentTablet.Shard != shr.Target.Shard { + if currentTarget.TabletType != shr.Target.TabletType || currentTarget.Keyspace != shr.Target.Keyspace || currentTarget.Shard != shr.Target.Shard { // keyspace and shard are not expected to change, but just in case ... // hc still has this tabletHealthCheck in the wrong target (because tabletType changed) - oldTargetKey := hc.keyFromTablet(currentTablet) + oldTargetKey := hc.keyFromTarget(currentTarget) newTargetKey := hc.keyFromTarget(shr.Target) - tabletAlias := topoproto.TabletAliasString(currentTablet.Alias) + tabletAlias := topoproto.TabletAliasString(shr.TabletAlias) hc.mu.Lock() delete(hc.healthData[oldTargetKey], tabletAlias) _, ok := hc.healthData[newTargetKey] @@ -324,8 +327,8 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str // need to replace it. if th.MasterTermStartTime < hc.healthy[targetKey][0].MasterTermStartTime { log.Warningf("not marking healthy master %s as Up for %s because its MasterTermStartTime is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", - topoproto.TabletAliasString(currentTablet.Alias), - topoproto.KeyspaceShardString(currentTablet.Keyspace, currentTablet.Shard), + topoproto.TabletAliasString(shr.TabletAlias), + topoproto.KeyspaceShardString(shr.Target.Keyspace, shr.Target.Shard), topoproto.TabletAliasString(hc.healthy[targetKey][0].Tablet.Alias), th.MasterTermStartTime, hc.healthy[targetKey][0].MasterTermStartTime) @@ -336,12 +339,16 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str } } + // notify downstream for master change result := th.simpleCopyLocked() - hc.broadcast(result) - // and notify downstream for master change - if shr.Target.TabletType == topodata.TabletType_MASTER && hc.masterCallback != nil { - hc.masterCallback(result) + if isMasterChange { + if hc.masterCallback != nil { + hc.masterCallback(result) + } + log.Errorf("Adding 1 to MasterPromoted counter for tablet: %v, shr.Tablet: %v, shr.TabletType: %v", currentTarget, topoproto.TabletAliasString(shr.TabletAlias), shr.Target.TabletType) + hcMasterPromotedCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard}, 1) } - + // broadcast to subscribers + hc.broadcast(result) return nil } diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index ffae58d4625..4e8433dfddb 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -868,7 +868,7 @@ func (e *Executor) handleShow(ctx context.Context, safeSession *SafeSession, sql } } if *GatewayImplementation == tabletGatewayImplementation { - stats := e.scatterConn.GetLegacyHealthCheckCacheStatus() + stats := e.scatterConn.GetHealthCheckCacheStatus() for _, s := range stats { for _, ts := range s.TabletsStats { state := "SERVING" From 3c89878ea33a05bd2001d3e13d673098c99b7db7 Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 12 May 2020 11:02:21 -0700 Subject: [PATCH 25/39] healthcheck: lock hc.mu before accessing members Signed-off-by: deepthi --- go/vt/discovery/tablet_health.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 7964eff5aef..8a158c3c0cf 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -310,6 +310,8 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str } th.setServingState(serving, reason) + hc.mu.Lock() + defer hc.mu.Unlock() targetKey := hc.keyFromTarget(shr.Target) if !trivialNonMasterUpdate { all := hc.healthData[targetKey] From 08250b810184e5e0daec9f84537604444cc716ee Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 12 May 2020 19:28:38 -0700 Subject: [PATCH 26/39] healthcheck: unit tests for GetHealthyTabletStats Signed-off-by: deepthi --- go/vt/discovery/healthcheck_test.go | 269 ++++++++++++++++++++++------ go/vt/discovery/tablet_health.go | 32 ++-- 2 files changed, 235 insertions(+), 66 deletions(-) diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go index b595f269739..b7e33488035 100644 --- a/go/vt/discovery/healthcheck_test.go +++ b/go/vt/discovery/healthcheck_test.go @@ -62,13 +62,11 @@ func TestHealthCheck(t *testing.T) { tablet.Type = topodatapb.TabletType_REPLICA input := make(chan *querypb.StreamHealthResponse) conn := createFakeConn(tablet, input) - t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) // create a channel and subscribe to healthcheck resultChan := hc.Subscribe() testChecksum(t, 0, hc.stateChecksum()) hc.AddTablet(tablet) - t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) testChecksum(t, 1027934207, hc.stateChecksum()) // Immediately after AddTablet() there will be the first notification. @@ -89,7 +87,6 @@ func TestHealthCheck(t *testing.T) { RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5}, } input <- shr - t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.5}}`) result = <-resultChan want = &TabletHealth{ Tablet: tablet, @@ -160,7 +157,6 @@ func TestHealthCheck(t *testing.T) { MasterTermStartTime: 0, } input <- shr - t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.3}}`) result = <-resultChan assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) testChecksum(t, 1027934207, hc.stateChecksum()) @@ -181,17 +177,13 @@ func TestHealthCheck(t *testing.T) { LastError: fmt.Errorf("vttablet error: some error"), } input <- shr - t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3}}`) result = <-resultChan assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) - testChecksum(t, 1027934207, hc.stateChecksum()) // unchanged // remove tablet hc.deleteConn(tablet) - t.Logf(`hc.RemoveTablet({Host: "a", PortMap: {"vt": 1}})`) testChecksum(t, 0, hc.stateChecksum()) - } func TestHealthCheckStreamError(t *testing.T) { @@ -205,10 +197,7 @@ func TestHealthCheckStreamError(t *testing.T) { resultChan := hc.Subscribe() fc := createFakeConn(tablet, input) fc.errCh = make(chan error) - t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) - hc.AddTablet(tablet) - t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. want := &TabletHealth{ @@ -235,7 +224,6 @@ func TestHealthCheckStreamError(t *testing.T) { MasterTermStartTime: 0, } input <- shr - t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) result = <-resultChan assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) @@ -262,11 +250,9 @@ func TestHealthCheckVerifiesTabletAlias(t *testing.T) { tablet.PortMap["vt"] = 1 input := make(chan *querypb.StreamHealthResponse, 1) fc := createFakeConn(tablet, input) - t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) resultChan := hc.Subscribe() hc.AddTablet(tablet) - t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. want := &TabletHealth{ @@ -289,7 +275,6 @@ func TestHealthCheckVerifiesTabletAlias(t *testing.T) { ticker := time.NewTicker(1 * time.Second) select { case err := <-fc.cbErrCh: - t.Logf("<-fc.cbErrCh: %v", err) assert.Contains(t, err.Error(), "health stats mismatch", "wrong error") case <-resultChan: require.Fail(t, "StreamHealth should have returned a health stats mismatch error") @@ -307,11 +292,9 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { tablet.PortMap["vt"] = 1 input := make(chan *querypb.StreamHealthResponse, 1) createFakeConn(tablet, input) - t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) resultChan := hc.Subscribe() hc.AddTablet(tablet) - t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. want := &TabletHealth{ @@ -338,7 +321,6 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { MasterTermStartTime: 0, } input <- shr - t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) result = <-resultChan assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) @@ -346,16 +328,13 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { shr.TabletExternallyReparentedTimestamp = 11 // Close the healthcheck. Tablet connections are closed asynchronously and // Close() will block until all Go routines (one per connection) are done. - hc.Close() + assert.Nil(t, hc.Close(), "Close returned error") // Try to send more updates. They should be ignored and nothing should change - // Note that this code is racy by nature. If there is a regression, it should - // fail in some cases. input <- shr - t.Logf(`input <- %v`, shr) select { case result = <-resultChan: - assert.Nil(t, result, "healthCheck still running after Close(): listener received: %v but should not have been called", result) + assert.Nil(t, result, "healthCheck still running after Close(): received result: %v", result) case <-time.After(1 * time.Millisecond): // No response after timeout. Success. } @@ -375,11 +354,8 @@ func TestHealthCheckTimeout(t *testing.T) { tablet.PortMap["vt"] = 1 input := make(chan *querypb.StreamHealthResponse) fc := createFakeConn(tablet, input) - t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`) resultChan := hc.Subscribe() - hc.AddTablet(tablet) - t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) // Immediately after AddTablet() there will be the first notification. want := &TabletHealth{ Tablet: tablet, @@ -405,58 +381,240 @@ func TestHealthCheckTimeout(t *testing.T) { MasterTermStartTime: 0, } input <- shr - t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) result = <-resultChan assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) - - if err := checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 0); err != nil { - t.Errorf("%v", err) - } + assert.Nil(t, checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 0)) // wait for timeout period time.Sleep(hc.healthCheckTimeout + 100*time.Millisecond) t.Logf(`Sleep(1.1 * timeout)`) result = <-resultChan - if result.Serving { - t.Errorf(`tabletHealthCheck: %+v; want not serving`, result) - } - - if err := checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 1); err != nil { - t.Errorf("%v", err) - } - - if !fc.isCanceled() { - t.Errorf("StreamHealth should be canceled after timeout, but is not") - } + assert.False(t, result.Serving, "tabletHealthCheck: %+v; want not serving", result) + assert.Nil(t, checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 1)) + assert.True(t, fc.isCanceled(), "StreamHealth should be canceled after timeout, but is not") // repeat the wait. It will timeout one more time trying to get the connection. fc.resetCanceledFlag() time.Sleep(hc.healthCheckTimeout) - t.Logf(`Sleep(timeout)`) result = <-resultChan - if result.Serving { - t.Errorf(`tabletHealthCheck: %+v; want not serving`, result) - } - - if err := checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 2); err != nil { - t.Errorf("%v", err) - } - - if !fc.isCanceled() { - t.Errorf("StreamHealth should be canceled again after timeout") - } + assert.False(t, result.Serving, "tabletHealthCheck: %+v; want not serving", result) + assert.Nil(t, checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 2)) + assert.True(t, fc.isCanceled(), "StreamHealth should be canceled again after timeout, but is not") // send a healthcheck response, it should be serving again fc.resetCanceledFlag() input <- shr - t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`) // wait for the exponential backoff to wear off and health monitoring to resume. result = <-resultChan assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) } +// TestGetHealthyTablets tests the functionality of GetHealthyTabletStats. +func TestGetHealthyTablets(t *testing.T) { + ts := memorytopo.NewServer("cell") + hc := createTestHc(ts) + + tablet := topo.NewTablet(0, "cell", "a") + tablet.Keyspace = "k" + tablet.Shard = "s" + tablet.PortMap["vt"] = 1 + tablet.Type = topodatapb.TabletType_REPLICA + input := make(chan *querypb.StreamHealthResponse) + createFakeConn(tablet, input) + + // create a channel and subscribe to healthcheck + resultChan := hc.Subscribe() + hc.AddTablet(tablet) + // there will be a first result, get and discard it + <-resultChan + // empty + a := hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) + assert.Equal(t, 0, len(a), "wrong result, expected empty list") + + shr := &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + want := &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + MasterTermStartTime: 0, + } + input <- shr + <-resultChan + // check it's there + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.Equal(t, 1, len(a), "Wrong number of results") + assert.True(t, want.DeepEqual(a[0]), "unexpected result") + + // update health with a change that won't change health array + shr = &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 2, CpuUsage: 0.2}, + } + input <- shr + // wait for result before checking + <-resultChan + // check it's there + want = &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 2, CpuUsage: 0.2}, + MasterTermStartTime: 0, + } + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.Equal(t, 1, len(a), "Wrong number of results") + assert.True(t, want.DeepEqual(a[0]), "unexpected result") + + // update stats with a change that will change health array + shr = &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 35, CpuUsage: 0.2}, + } + want = &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 35, CpuUsage: 0.2}, + MasterTermStartTime: 0, + } + input <- shr + // wait for result before checking + <-resultChan + // check it's there + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.Equal(t, 1, len(a), "Wrong number of results") + assert.True(t, want.DeepEqual(a[0]), "unexpected result") + + // add a second tablet + tablet2 := topo.NewTablet(11, "cell", "host2") + tablet2.Keyspace = "k" + tablet2.Shard = "s" + tablet2.PortMap["vt"] = 2 + tablet2.Type = topodatapb.TabletType_REPLICA + input2 := make(chan *querypb.StreamHealthResponse) + createFakeConn(tablet2, input2) + t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 2}}, c)`) + hc.AddTablet(tablet2) + t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`) + // there will be a first result, get and discard it + <-resultChan + + shr2 := &querypb.StreamHealthResponse{ + TabletAlias: tablet2.Alias, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, + } + want2 := &TabletHealth{ + Tablet: tablet2, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, + MasterTermStartTime: 0, + } + input2 <- shr2 + // wait for result + <-resultChan + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.Equal(t, 2, len(a), "Wrong number of results") + if a[0].Tablet.Alias.Uid == 11 { + a[0], a[1] = a[1], a[0] + } + assert.True(t, want.DeepEqual(a[0]), "unexpected result") + assert.True(t, want2.DeepEqual(a[1]), "unexpected result") + + shr2 = &querypb.StreamHealthResponse{ + TabletAlias: tablet2.Alias, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: false, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, + } + input2 <- shr2 + // wait for result + <-resultChan + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.Equal(t, 1, len(a), "Wrong number of results") + assert.True(t, want.DeepEqual(a[0]), "unexpected result") + + // second tablet turns into a master + shr2 = &querypb.StreamHealthResponse{ + TabletAlias: tablet2.Alias, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, + Serving: true, + TabletExternallyReparentedTimestamp: 10, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 0, CpuUsage: 0.2}, + } + input2 <- shr2 + // wait for result + <-resultChan + // check we only have 1 healthy replica left + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.Equal(t, 1, len(a), "Wrong number of results") + assert.True(t, want.DeepEqual(a[0]), "unexpected result") + + want2 = &TabletHealth{ + Tablet: tablet2, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 0, CpuUsage: 0.2}, + MasterTermStartTime: 10, + } + // check we have a master now + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) + assert.Equal(t, 1, len(a), "Wrong number of results") + assert.True(t, want2.DeepEqual(a[0]), "unexpected result") + + // reparent: old replica goes into master + shr = &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, + Serving: true, + TabletExternallyReparentedTimestamp: 20, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 0, CpuUsage: 0.2}, + } + input <- shr + <-resultChan + want = &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 0, CpuUsage: 0.2}, + MasterTermStartTime: 20, + } + + // check we lost all replicas, and master is new one + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.Equal(t, 0, len(a), "Wrong number of results") + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) + assert.Equal(t, 1, len(a), "Wrong number of results") + assert.True(t, want.DeepEqual(a[0]), "unexpected result") + + // old master sending an old ping should be ignored + input2 <- shr2 + <-resultChan + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) + assert.Equal(t, 1, len(a), "Wrong number of results") + assert.True(t, want.DeepEqual(a[0]), "unexpected result") + assert.Nil(t, hc.Close(), "unexpected error") +} + func TestTemplate(t *testing.T) { tablet := topo.NewTablet(0, "cell", "a") ts := []*TabletHealth{ @@ -601,6 +759,5 @@ func checkErrorCounter(keyspace, shard string, tabletType topodatapb.TabletType, if got != want { return fmt.Errorf("wrong value for hcErrorCounters got = %v, want = %v", got, want) } - return nil } diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 8a158c3c0cf..c011bd66acc 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -280,9 +280,10 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str hc.mu.Unlock() hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) - if currentTarget.TabletType != shr.Target.TabletType || currentTarget.Keyspace != shr.Target.Keyspace || currentTarget.Shard != shr.Target.Shard { + targetChanged := currentTarget.TabletType != shr.Target.TabletType || currentTarget.Keyspace != shr.Target.Keyspace || currentTarget.Shard != shr.Target.Shard + if targetChanged { // keyspace and shard are not expected to change, but just in case ... - // hc still has this tabletHealthCheck in the wrong target (because tabletType changed) + // move this tabletHealthCheck to the correct map oldTargetKey := hc.keyFromTarget(currentTarget) newTargetKey := hc.keyFromTarget(shr.Target) tabletAlias := topoproto.TabletAliasString(shr.TabletAlias) @@ -313,14 +314,6 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str hc.mu.Lock() defer hc.mu.Unlock() targetKey := hc.keyFromTarget(shr.Target) - if !trivialNonMasterUpdate { - all := hc.healthData[targetKey] - allArray := make([]*tabletHealthCheck, 0, len(all)) - for _, s := range all { - allArray = append(allArray, s) - } - hc.healthy[targetKey] = FilterStatsByReplicationLag(allArray) - } if isMasterUpdate { if len(hc.healthy[targetKey]) == 0 { hc.healthy[targetKey] = append(hc.healthy[targetKey], th) @@ -340,6 +333,25 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str } } } + if !trivialNonMasterUpdate { + if shr.Target.TabletType != topodata.TabletType_MASTER { + all := hc.healthData[targetKey] + allArray := make([]*tabletHealthCheck, 0, len(all)) + for _, s := range all { + allArray = append(allArray, s) + } + hc.healthy[targetKey] = FilterStatsByReplicationLag(allArray) + } + if targetChanged && currentTarget.TabletType != topodata.TabletType_MASTER { // also recompute old target's healthy list + oldTargetKey := hc.keyFromTarget(currentTarget) + all := hc.healthData[oldTargetKey] + allArray := make([]*tabletHealthCheck, 0, len(all)) + for _, s := range all { + allArray = append(allArray, s) + } + hc.healthy[oldTargetKey] = FilterStatsByReplicationLag(allArray) + } + } // notify downstream for master change result := th.simpleCopyLocked() From 00d137edec47992e2c6ce4c9f5ece1b4854785f5 Mon Sep 17 00:00:00 2001 From: deepthi Date: Wed, 13 May 2020 11:55:17 -0700 Subject: [PATCH 27/39] healthcheck: update protobuf generated sources, fix shadowing bug which results in incomplete error information Signed-off-by: deepthi --- go/vt/proto/query/query.pb.go | 2 +- go/vt/vtgate/tabletgateway.go | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/go/vt/proto/query/query.pb.go b/go/vt/proto/query/query.pb.go index 260948bfaa4..45f9128cebd 100644 --- a/go/vt/proto/query/query.pb.go +++ b/go/vt/proto/query/query.pb.go @@ -3629,7 +3629,7 @@ type StreamHealthResponse struct { // realtime_stats contains information about the tablet status. // It is only filled in if the information is about a tablet. RealtimeStats *RealtimeStats `protobuf:"bytes,4,opt,name=realtime_stats,json=realtimeStats,proto3" json:"realtime_stats,omitempty"` - // tablet_alias is the alias of the sending tablet. The discovery/legacy_healthcheck.go + // tablet_alias is the alias of the sending tablet. The discovery/healthcheck.go // code uses it to verify that it's talking to the correct tablet and that it // hasn't changed in the meantime e.g. due to tablet restarts where ports or // ips have been reused but assigned differently. diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index 1ff6d1204b9..dca926b8904 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -200,19 +200,18 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, } gw.shuffleTablets(gw.localCell, tablets) - var tabletLastUsed string var th *discovery.TabletHealth // skip tablets we tried before for _, t := range tablets { - tabletLastUsed = topoproto.TabletAliasString(t.Tablet.Alias) - if _, ok := invalidTablets[tabletLastUsed]; !ok { + tabletLastUsed = t.Tablet + if _, ok := invalidTablets[topoproto.TabletAliasString(tabletLastUsed.Alias)]; !ok { th = t break } else { - tabletLastUsed = "" + tabletLastUsed = nil } } - if tabletLastUsed == "" { + if tabletLastUsed == nil { // do not override error from last attempt. if err == nil { err = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no available connection") @@ -223,7 +222,7 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, // execute if th.Conn == nil { err = vterrors.Errorf(vtrpcpb.Code_UNAVAILABLE, "no connection for tablet %v", tabletLastUsed) - invalidTablets[tabletLastUsed] = true + invalidTablets[topoproto.TabletAliasString(tabletLastUsed.Alias)] = true continue } @@ -232,7 +231,7 @@ func (gw *TabletGateway) withRetry(ctx context.Context, target *querypb.Target, canRetry, err = inner(ctx, target, th.Conn) gw.updateStats(target, startTime, err) if canRetry { - invalidTablets[tabletLastUsed] = true + invalidTablets[topoproto.TabletAliasString(tabletLastUsed.Alias)] = true continue } break From c4fab257dc64b01d75f91e4ca959f2730d289e62 Mon Sep 17 00:00:00 2001 From: deepthi Date: Wed, 13 May 2020 16:25:39 -0700 Subject: [PATCH 28/39] healthcheck: cell alias test, add new tests to config.json Signed-off-by: deepthi --- .../buffer/buffer_test.go | 0 .../cellalias/cell_alias_test.go | 370 ++++++++++++++++++ .../healthcheck/main_test.go | 0 .../healthcheck/vtgate_test.go | 0 go/vt/discovery/healthcheck.go | 39 ++ go/vt/discovery/topology_watcher.go | 31 +- test/config.json | 27 ++ 7 files changed, 437 insertions(+), 30 deletions(-) rename go/test/endtoend/{vtgate/healthcheck => tabletgateway}/buffer/buffer_test.go (100%) create mode 100644 go/test/endtoend/tabletgateway/cellalias/cell_alias_test.go rename go/test/endtoend/{vtgate => tabletgateway}/healthcheck/main_test.go (100%) rename go/test/endtoend/{vtgate => tabletgateway}/healthcheck/vtgate_test.go (100%) diff --git a/go/test/endtoend/vtgate/healthcheck/buffer/buffer_test.go b/go/test/endtoend/tabletgateway/buffer/buffer_test.go similarity index 100% rename from go/test/endtoend/vtgate/healthcheck/buffer/buffer_test.go rename to go/test/endtoend/tabletgateway/buffer/buffer_test.go diff --git a/go/test/endtoend/tabletgateway/cellalias/cell_alias_test.go b/go/test/endtoend/tabletgateway/cellalias/cell_alias_test.go new file mode 100644 index 00000000000..089ec7a8666 --- /dev/null +++ b/go/test/endtoend/tabletgateway/cellalias/cell_alias_test.go @@ -0,0 +1,370 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This test cell aliases feature + +We start with no aliases and assert that vtgates can't route to replicas/rondly tablets. +Then we add an alias, and these tablets should be routable +*/ + +package binlog + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/test/endtoend/sharding" + "vitess.io/vitess/go/vt/proto/topodata" +) + +var ( + localCluster *cluster.LocalProcessCluster + cell1 = "zone1" + cell2 = "zone2" + hostname = "localhost" + keyspaceName = "ks" + tableName = "test_table" + sqlSchema = ` + create table %s( + id bigint(20) unsigned auto_increment, + msg varchar(64), + primary key (id), + index by_msg (msg) + ) Engine=InnoDB +` + commonTabletArg = []string{ + "-vreplication_healthcheck_topology_refresh", "1s", + "-vreplication_healthcheck_retry_delay", "1s", + "-vreplication_retry_delay", "1s", + "-degraded_threshold", "5s", + "-lock_tables_timeout", "5s", + "-watch_replication_stream", + "-enable_replication_reporter", + "-serving_state_grace_period", "1s", + "-binlog_player_protocol", "grpc", + "-enable-autocommit", + } + vSchema = ` + { + "sharded": true, + "vindexes": { + "hash_index": { + "type": "hash" + } + }, + "tables": { + "%s": { + "column_vindexes": [ + { + "column": "id", + "name": "hash_index" + } + ] + } + } + } +` + shard1Master *cluster.Vttablet + shard1Replica *cluster.Vttablet + shard1Rdonly *cluster.Vttablet + shard2Master *cluster.Vttablet + shard2Replica *cluster.Vttablet + shard2Rdonly *cluster.Vttablet +) + +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + flag.Parse() + + exitcode, err := func() (int, error) { + localCluster = cluster.NewCluster(cell1, hostname) + defer localCluster.Teardown() + localCluster.Keyspaces = append(localCluster.Keyspaces, cluster.Keyspace{ + Name: keyspaceName, + }) + + // Start topo server + if err := localCluster.StartTopo(); err != nil { + return 1, err + } + + // Adding another cell in the same cluster + err := localCluster.TopoProcess.ManageTopoDir("mkdir", "/vitess/"+cell2) + if err != nil { + return 1, err + } + err = localCluster.VtctlProcess.AddCellInfo(cell2) + if err != nil { + return 1, err + } + + shard1Master = localCluster.GetVttabletInstance("master", 0, "") + shard1Replica = localCluster.GetVttabletInstance("replica", 0, cell2) + shard1Rdonly = localCluster.GetVttabletInstance("rdonly", 0, cell2) + + shard2Master = localCluster.GetVttabletInstance("master", 0, "") + shard2Replica = localCluster.GetVttabletInstance("replica", 0, cell2) + shard2Rdonly = localCluster.GetVttabletInstance("rdonly", 0, cell2) + + var mysqlProcs []*exec.Cmd + for _, tablet := range []*cluster.Vttablet{shard1Master, shard1Replica, shard1Rdonly, shard2Master, shard2Replica, shard2Rdonly} { + tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) + tablet.VttabletProcess = cluster.VttabletProcessInstance(tablet.HTTPPort, + tablet.GrpcPort, + tablet.TabletUID, + tablet.Cell, + "", + keyspaceName, + localCluster.VtctldProcess.Port, + tablet.Type, + localCluster.TopoPort, + hostname, + localCluster.TmpDirectory, + commonTabletArg, + true, + ) + tablet.VttabletProcess.SupportsBackup = true + proc, err := tablet.MysqlctlProcess.StartProcess() + if err != nil { + return 1, err + } + mysqlProcs = append(mysqlProcs, proc) + } + for _, proc := range mysqlProcs { + if err := proc.Wait(); err != nil { + return 1, err + } + } + + if err := localCluster.VtctlProcess.CreateKeyspace(keyspaceName); err != nil { + return 1, err + } + + shard1 := cluster.Shard{ + Name: "-80", + Vttablets: []*cluster.Vttablet{shard1Master, shard1Replica, shard1Rdonly}, + } + for idx := range shard1.Vttablets { + shard1.Vttablets[idx].VttabletProcess.Shard = shard1.Name + } + localCluster.Keyspaces[0].Shards = append(localCluster.Keyspaces[0].Shards, shard1) + + shard2 := cluster.Shard{ + Name: "80-", + Vttablets: []*cluster.Vttablet{shard2Master, shard2Replica, shard2Rdonly}, + } + for idx := range shard2.Vttablets { + shard2.Vttablets[idx].VttabletProcess.Shard = shard2.Name + } + localCluster.Keyspaces[0].Shards = append(localCluster.Keyspaces[0].Shards, shard2) + + for _, tablet := range shard1.Vttablets { + if err := localCluster.VtctlclientProcess.InitTablet(tablet, tablet.Cell, keyspaceName, hostname, shard1.Name); err != nil { + return 1, err + } + if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil { + return 1, err + } + if err := tablet.VttabletProcess.Setup(); err != nil { + return 1, err + } + } + if err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shard1.Name, shard1Master.Cell, shard1Master.TabletUID); err != nil { + return 1, err + } + + // run a health check on source replica so it responds to discovery + // (for binlog players) and on the source rdonlys (for workers) + for _, tablet := range []string{shard1Replica.Alias, shard1Rdonly.Alias} { + if err := localCluster.VtctlclientProcess.ExecuteCommand("RunHealthCheck", tablet); err != nil { + return 1, err + } + } + + for _, tablet := range shard2.Vttablets { + if err := localCluster.VtctlclientProcess.InitTablet(tablet, tablet.Cell, keyspaceName, hostname, shard2.Name); err != nil { + return 1, err + } + if err := tablet.VttabletProcess.CreateDB(keyspaceName); err != nil { + return 1, err + } + if err := tablet.VttabletProcess.Setup(); err != nil { + return 1, err + } + } + + if err := localCluster.VtctlclientProcess.InitShardMaster(keyspaceName, shard2.Name, shard2Master.Cell, shard2Master.TabletUID); err != nil { + return 1, err + } + + if err := localCluster.VtctlclientProcess.ApplySchema(keyspaceName, fmt.Sprintf(sqlSchema, tableName)); err != nil { + return 1, err + } + if err := localCluster.VtctlclientProcess.ApplyVSchema(keyspaceName, fmt.Sprintf(vSchema, tableName)); err != nil { + return 1, err + } + + _ = localCluster.VtctlclientProcess.ExecuteCommand("RebuildKeyspaceGraph", keyspaceName) + + return m.Run(), nil + }() + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } else { + os.Exit(exitcode) + } +} + +func TestAlias(t *testing.T) { + defer cluster.PanicHandler(t) + + insertInitialValues(t) + defer deleteInitialValues(t) + + err := localCluster.VtctlclientProcess.ExecuteCommand("RebuildKeyspaceGraph", keyspaceName) + require.Nil(t, err) + shard1 := localCluster.Keyspaces[0].Shards[0] + shard2 := localCluster.Keyspaces[0].Shards[1] + allCells := fmt.Sprintf("%s,%s", cell1, cell2) + + expectedPartitions := map[topodata.TabletType][]string{} + expectedPartitions[topodata.TabletType_MASTER] = []string{shard1.Name, shard2.Name} + expectedPartitions[topodata.TabletType_REPLICA] = []string{shard1.Name, shard2.Name} + expectedPartitions[topodata.TabletType_RDONLY] = []string{shard1.Name, shard2.Name} + sharding.CheckSrvKeyspace(t, cell1, keyspaceName, "", 0, expectedPartitions, *localCluster) + sharding.CheckSrvKeyspace(t, cell2, keyspaceName, "", 0, expectedPartitions, *localCluster) + + // Adds alias so vtgate can route to replica/rdonly tablets that are not in the same cell, but same alias + err = localCluster.VtctlclientProcess.ExecuteCommand("AddCellsAlias", + "-cells", allCells, + "region_east_coast") + require.Nil(t, err) + err = localCluster.VtctlclientProcess.ExecuteCommand("UpdateCellsAlias", + "-cells", allCells, + "region_east_coast") + require.Nil(t, err) + + vtgateInstance := localCluster.GetVtgateInstance() + vtgateInstance.CellsToWatch = allCells + vtgateInstance.TabletTypesToWait = "MASTER,REPLICA" + vtgateInstance.GatewayImplementation = "tabletgateway" + err = vtgateInstance.Setup() + require.Nil(t, err) + + // Cluster teardown will not teardown vtgate because we are not + // actually setting this on localCluster.VtgateInstance + defer vtgateInstance.TearDown() + + waitTillAllTabletsAreHealthyInVtgate(t, *vtgateInstance, shard1.Name, shard2.Name) + + testQueriesOnTabletType(t, "master", vtgateInstance.GrpcPort, false) + testQueriesOnTabletType(t, "replica", vtgateInstance.GrpcPort, false) + testQueriesOnTabletType(t, "rdonly", vtgateInstance.GrpcPort, false) + + // now, delete the alias, so that if we run above assertions again, it will fail for replica,rdonly target type + err = localCluster.VtctlclientProcess.ExecuteCommand("DeleteCellsAlias", + "region_east_coast") + require.Nil(t, err) + + // restarts the vtgate process + vtgateInstance.TabletTypesToWait = "MASTER" + err = vtgateInstance.TearDown() + require.Nil(t, err) + err = vtgateInstance.Setup() + require.Nil(t, err) + + // since replica and rdonly tablets of all shards in cell2, the last 2 assertion is expected to fail + testQueriesOnTabletType(t, "master", vtgateInstance.GrpcPort, false) + testQueriesOnTabletType(t, "replica", vtgateInstance.GrpcPort, true) + testQueriesOnTabletType(t, "rdonly", vtgateInstance.GrpcPort, true) + +} + +func waitTillAllTabletsAreHealthyInVtgate(t *testing.T, vtgateInstance cluster.VtgateProcess, shards ...string) { + for _, shard := range shards { + err := vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.master", keyspaceName, shard), 1) + require.Nil(t, err) + err = vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.replica", keyspaceName, shard), 1) + require.Nil(t, err) + err = vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.rdonly", keyspaceName, shard), 1) + require.Nil(t, err) + } +} + +func testQueriesOnTabletType(t *testing.T, tabletType string, vtgateGrpcPort int, shouldFail bool) { + output, err := localCluster.VtctlProcess.ExecuteCommandWithOutput("VtGateExecute", "-json", + "-server", fmt.Sprintf("%s:%d", localCluster.Hostname, vtgateGrpcPort), + "-target", "@"+tabletType, + fmt.Sprintf(`select * from %s`, tableName)) + if shouldFail { + require.Error(t, err) + return + } + require.Nil(t, err) + var result sqltypes.Result + + err = json.Unmarshal([]byte(output), &result) + require.Nil(t, err) + assert.Equal(t, len(result.Rows), 3) +} + +func insertInitialValues(t *testing.T) { + sharding.ExecuteOnTablet(t, + fmt.Sprintf(sharding.InsertTabletTemplateKsID, tableName, 1, "msg1", 1), + *shard1Master, + keyspaceName, + false) + + sharding.ExecuteOnTablet(t, + fmt.Sprintf(sharding.InsertTabletTemplateKsID, tableName, 2, "msg2", 2), + *shard1Master, + keyspaceName, + false) + + sharding.ExecuteOnTablet(t, + fmt.Sprintf(sharding.InsertTabletTemplateKsID, tableName, 4, "msg4", 4), + *shard2Master, + keyspaceName, + false) +} + +func deleteInitialValues(t *testing.T) { + sharding.ExecuteOnTablet(t, + fmt.Sprintf("delete from %s where id = %v", tableName, 1), + *shard1Master, + keyspaceName, + false) + + sharding.ExecuteOnTablet(t, + fmt.Sprintf("delete from %s where id = %v", tableName, 2), + *shard1Master, + keyspaceName, + false) + + sharding.ExecuteOnTablet(t, + fmt.Sprintf("delete from %s where id = %v", tableName, 4), + *shard2Master, + keyspaceName, + false) +} diff --git a/go/test/endtoend/vtgate/healthcheck/main_test.go b/go/test/endtoend/tabletgateway/healthcheck/main_test.go similarity index 100% rename from go/test/endtoend/vtgate/healthcheck/main_test.go rename to go/test/endtoend/tabletgateway/healthcheck/main_test.go diff --git a/go/test/endtoend/vtgate/healthcheck/vtgate_test.go b/go/test/endtoend/tabletgateway/healthcheck/vtgate_test.go similarity index 100% rename from go/test/endtoend/vtgate/healthcheck/vtgate_test.go rename to go/test/endtoend/tabletgateway/healthcheck/vtgate_test.go diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 7462f17fe93..f30e429f2b6 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -209,6 +209,8 @@ type HealthCheckImpl struct { // used to inform vtgate buffer when new master is detected // TODO: replace this with synchronizing over a condition variable masterCallback func(health *TabletHealth) + // cellAliases is a cache of cell aliases + cellAliases map[string]string // mutex to protect subscribers subMu sync.Mutex // subscribers @@ -270,6 +272,7 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur healthData: make(map[string]map[string]*tabletHealthCheck), healthy: make(map[string][]*tabletHealthCheck), subscribers: make(map[chan *TabletHealth]struct{}), + cellAliases: make(map[string]string), } var topoWatchers []*TopologyWatcher var filter TabletFilter @@ -527,6 +530,10 @@ func (hc *HealthCheckImpl) deleteConn(tablet *topodata.Tablet) { // name is an optional tag for the tablet, e.g. an alternative address. func (hc *HealthCheckImpl) AddTablet(tablet *topodata.Tablet) { log.Infof("Calling AddTablet for tablet: %v", tablet) + // check whether we should really add this tablet + if !hc.isIncluded(tablet) { + return + } hc.mu.Lock() if hc.healthByAlias == nil { // already closed. @@ -574,6 +581,9 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodata.Tablet) { // RemoveTablet removes the tablet, and stops the health check. // It does not block. func (hc *HealthCheckImpl) RemoveTablet(tablet *topodata.Tablet) { + if !hc.isIncluded(tablet) { + return + } hc.deleteConn(tablet) } @@ -763,3 +773,32 @@ func (hc *HealthCheckImpl) keyFromTarget(target *query.Target) string { func (hc *HealthCheckImpl) keyFromTablet(tablet *topodata.Tablet) string { return fmt.Sprintf("%s.%s.%d", tablet.Keyspace, tablet.Shard, tablet.Type) } + +func (hc *HealthCheckImpl) getAliasByCell(cell string) string { + hc.mu.Lock() + defer hc.mu.Unlock() + + if alias, ok := hc.cellAliases[cell]; ok { + return alias + } + + alias := topo.GetAliasByCell(context.Background(), hc.ts, cell) + // Currently cell aliases have to be non-overlapping. + // If that changes, this will need to change to account for overlaps. + hc.cellAliases[cell] = alias + + return alias +} + +func (hc *HealthCheckImpl) isIncluded(tablet *topodata.Tablet) bool { + if tablet.Type == topodata.TabletType_MASTER { + return true + } + if tablet.Alias.Cell == hc.cell { + return true + } + if hc.getAliasByCell(tablet.Alias.Cell) == hc.getAliasByCell(hc.cell) { + return true + } + return false +} diff --git a/go/vt/discovery/topology_watcher.go b/go/vt/discovery/topology_watcher.go index a4d92af915a..802ec4d4a83 100644 --- a/go/vt/discovery/topology_watcher.go +++ b/go/vt/discovery/topology_watcher.go @@ -89,8 +89,6 @@ type TopologyWatcher struct { firstLoadDone bool // firstLoadChan is closed when the initial loading of topology data is done. firstLoadChan chan struct{} - // cellAliases is a cache of cell aliases - cellAliases map[string]string } // NewTopologyWatcher returns a TopologyWatcher that monitors all @@ -198,7 +196,7 @@ func (tw *TopologyWatcher) loadTablets() { log.Errorf("cannot get tablet for alias %v: %v", alias, err) return } - if !(tw.isTabletInCell(tablet.Tablet) && (tw.tabletFilter == nil || tw.tabletFilter.IsIncluded(tablet.Tablet))) { + if !(tw.tabletFilter == nil || tw.tabletFilter.IsIncluded(tablet.Tablet)) { return } tw.mu.Lock() @@ -262,33 +260,6 @@ func (tw *TopologyWatcher) loadTablets() { } -func (tw *TopologyWatcher) getAliasByCell(cell string) string { - tw.mu.Lock() - defer tw.mu.Unlock() - - if alias, ok := tw.cellAliases[cell]; ok { - return alias - } - - alias := topo.GetAliasByCell(context.Background(), tw.topoServer, cell) - tw.cellAliases[cell] = alias - - return alias -} - -func (tw *TopologyWatcher) isTabletInCell(tablet *topodata.Tablet) bool { - if tablet.Type == topodata.TabletType_MASTER { - return true - } - if tablet.Alias.Cell == tw.cell { - return true - } - if tw.getAliasByCell(tablet.Alias.Cell) == tw.getAliasByCell(tw.cell) { - return true - } - return false -} - // RefreshLag returns the time since the last refresh func (tw *TopologyWatcher) RefreshLag() time.Duration { tw.mu.Lock() diff --git a/test/config.json b/test/config.json index dfc4d677438..67fad2afdce 100644 --- a/test/config.json +++ b/test/config.json @@ -324,6 +324,33 @@ "RetryMax": 0, "Tags": [] }, + "tabletgateway_buffer": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/tabletgateway/buffer"], + "Command": [], + "Manual": false, + "Shard": 14, + "RetryMax": 0, + "Tags": [] + }, + "tabletgateway_cellalias": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/tabletgateway/cellalias"], + "Command": [], + "Manual": false, + "Shard": 14, + "RetryMax": 0, + "Tags": [] + }, + "tabletgateway_healthcheck": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/tabletgateway/healthcheck"], + "Command": [], + "Manual": false, + "Shard": 14, + "RetryMax": 0, + "Tags": [] + }, "tabletmanager": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/tabletmanager"], From 97a77a457bee0eecb578532acd66e4a8671a893d Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 14 May 2020 09:42:56 -0700 Subject: [PATCH 29/39] healthcheck: cell alias unit test Signed-off-by: deepthi --- go/vt/discovery/healthcheck_test.go | 96 ++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go index b7e33488035..91dc105f6f1 100644 --- a/go/vt/discovery/healthcheck_test.go +++ b/go/vt/discovery/healthcheck_test.go @@ -415,7 +415,7 @@ func TestHealthCheckTimeout(t *testing.T) { func TestGetHealthyTablets(t *testing.T) { ts := memorytopo.NewServer("cell") hc := createTestHc(ts) - + defer hc.Close() tablet := topo.NewTablet(0, "cell", "a") tablet.Keyspace = "k" tablet.Shard = "s" @@ -612,7 +612,99 @@ func TestGetHealthyTablets(t *testing.T) { a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) assert.Equal(t, 1, len(a), "Wrong number of results") assert.True(t, want.DeepEqual(a[0]), "unexpected result") - assert.Nil(t, hc.Close(), "unexpected error") +} + +func TestAliases(t *testing.T) { + ts := memorytopo.NewServer("cell", "cell1", "cell2") + hc := createTestHc(ts) + defer hc.Close() + + cellsAlias := &topodatapb.CellsAlias{ + Cells: []string{"cell", "cell1"}, + } + assert.Nil(t, ts.CreateCellsAlias(context.Background(), "region1", cellsAlias), "failed to create cell alias") + defer ts.DeleteCellsAlias(context.Background(), "region1") + cellsAlias = &topodatapb.CellsAlias{ + Cells: []string{"cell2"}, + } + assert.Nil(t, ts.CreateCellsAlias(context.Background(), "region2", cellsAlias), "failed to create cell alias") + defer ts.DeleteCellsAlias(context.Background(), "region2") + + // add a tablet as replica in diff cell, same region + tablet := topo.NewTablet(1, "cell1", "host3") + tablet.Keyspace = "k" + tablet.Shard = "s" + tablet.PortMap["vt"] = 1 + tablet.Type = topodatapb.TabletType_REPLICA + input := make(chan *querypb.StreamHealthResponse) + fc := createFakeConn(tablet, input) + // create a channel and subscribe to healthcheck + resultChan := hc.Subscribe() + hc.AddTablet(tablet) + // should get a result, but this will hang if cell alias logic is broken + // so wait and timeout + ticker := time.NewTicker(1 * time.Second) + select { + case err := <-fc.cbErrCh: + require.Fail(t, "Unexpected error: %v", err) + case <-resultChan: + case <-ticker.C: + require.Fail(t, "Timed out waiting for HealthCheck update") + } + + shr := &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + TabletExternallyReparentedTimestamp: 0, + RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, + } + want := &TabletHealth{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, + MasterTermStartTime: 0, + } + + input <- shr + ticker = time.NewTicker(1 * time.Second) + select { + case err := <-fc.cbErrCh: + require.Fail(t, "Unexpected error: %v", err) + case <-resultChan: + case <-ticker.C: + require.Fail(t, "Timed out waiting for HealthCheck update") + } + + // check it's there + a := hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.Equal(t, 1, len(a), "Wrong number of results") + assert.True(t, want.DeepEqual(a[0]), "unexpected result") + + // add another tablet in a diff cell, diff region + tablet2 := topo.NewTablet(2, "cell2", "host4") + tablet2.Keyspace = "k" + tablet2.Shard = "s" + tablet2.PortMap["vt"] = 2 + tablet2.Type = topodatapb.TabletType_REPLICA + input2 := make(chan *querypb.StreamHealthResponse) + fc = createFakeConn(tablet2, input2) + hc.AddTablet(tablet2) + // we should NOT get a result because this tablet is not of interest to us + ticker = time.NewTicker(1 * time.Second) + select { + case err := <-fc.cbErrCh: + require.Fail(t, "Unexpected error: %v", err) + case result := <-resultChan: + require.Fail(t, "Unexpected result: %v", result) + case <-ticker.C: + } + + // check that we still have only tablet in healthy list + a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) + assert.Equal(t, 1, len(a), "Wrong number of results") + assert.True(t, want.DeepEqual(a[0]), "unexpected result") } func TestTemplate(t *testing.T) { From e0fb77e3bad8209a81245a571bf892d4f274f5c5 Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 19 May 2020 22:13:55 -0700 Subject: [PATCH 30/39] healthcheck: remove HealthCheck interface, rename HealthCheckImpl -> HealthCheck, move checkConn and finalizeConn to tabletHealthCheck, rename deleteConn -> deleteTablet Signed-off-by: deepthi --- go/cmd/vtgate/vtgate.go | 6 + go/vt/discovery/healthcheck.go | 557 +++++++++++++--------------- go/vt/discovery/healthcheck_test.go | 13 +- go/vt/discovery/tablet_health.go | 278 +++++++------- go/vt/vtgate/api.go | 2 +- go/vt/vtgate/discoverygateway.go | 2 +- go/vt/vtgate/gateway.go | 2 +- go/vt/vtgate/tabletgateway.go | 4 +- 8 files changed, 425 insertions(+), 439 deletions(-) diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index 1c346be4e1a..8c696f97887 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -89,5 +89,11 @@ func main() { discovery.ParseTabletURLTemplateFromFlag() addStatusParts(vtg) }) + servenv.OnClose(func() { + _ = vtg.Gateway().Close(context.Background()) + if legacyHealthCheck != nil { + _ = legacyHealthCheck.Close() + } + }) servenv.RunDefault() } diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index f30e429f2b6..ee9c20a6334 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -50,7 +50,6 @@ import ( "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/stats" - "vitess.io/vitess/go/sync2" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/proto/topodata" @@ -158,26 +157,11 @@ type TabletRecorder interface { ReplaceTablet(old, new *topodata.Tablet) } -// HealthCheck defines the interface of health checking module. +// HealthCheck performs health checking and stores the results. // The goal of this object is to maintain a StreamHealth RPC // to a lot of tablets. Tablets are added / removed by calling the // AddTablet / RemoveTablet methods (other discovery module objects // can for instance watch the topology and call these). -type HealthCheck interface { - // RegisterStats registers the connection counts and checksum stats. - // It can only be called on one Healthcheck object per process. - RegisterStats() - // CacheStatus returns a displayable version of the cache. - CacheStatus() TabletsCacheStatusList - // Close stops the healthcheck. - Close() error - // GetHealthyTabletStatts - GetHealthyTabletStats(target *query.Target) []*TabletHealth - // WaitForAllServingTablets allows vtgate to wait for all tablets to be serving before accepting requests - WaitForAllServingTablets(ctx context.Context, targets []*query.Target) error -} - -// HealthCheckImpl performs health checking and stores the results. // It contains a map of tabletHealthCheck objects by Alias. // Each tabletHealthCheck object stores the health information for one tablet. // A checkConn goroutine is spawned for each tabletHealthCheck, which is responsible for @@ -186,7 +170,7 @@ type HealthCheck interface { // is removed from the map. When a tabletHealthCheck // gets removed from the map, its cancelFunc gets called, which ensures that the associated // checkConn goroutine eventually terminates. -type HealthCheckImpl struct { +type HealthCheck struct { // Immutable fields set at construction time. retryDelay time.Duration healthCheckTimeout time.Duration @@ -207,7 +191,7 @@ type HealthCheckImpl struct { // topology watchers that inform healthcheck of tablets being added and deleted topoWatchers []*TopologyWatcher // used to inform vtgate buffer when new master is detected - // TODO: replace this with synchronizing over a condition variable + // TODO: buffer should subscribe to healthcheck instead of setting a callback masterCallback func(health *TabletHealth) // cellAliases is a cache of cell aliases cellAliases map[string]string @@ -217,33 +201,6 @@ type HealthCheckImpl struct { subscribers map[chan *TabletHealth]struct{} } -// Subscribe adds a listener. Only used for testing right now -func (hc *HealthCheckImpl) Subscribe() chan *TabletHealth { - hc.subMu.Lock() - defer hc.subMu.Unlock() - c := make(chan *TabletHealth, 2) - hc.subscribers[c] = struct{}{} - return c -} - -// Unsubscribe removes a listener. Only used for testing right now -func (hc *HealthCheckImpl) Unsubscribe(c chan *TabletHealth) { - hc.subMu.Lock() - defer hc.subMu.Unlock() - delete(hc.subscribers, c) -} - -func (hc *HealthCheckImpl) broadcast(th *TabletHealth) { - hc.subMu.Lock() - defer hc.subMu.Unlock() - for c := range hc.subscribers { - select { - case c <- th: - default: - } - } -} - // NewHealthCheck creates a new HealthCheck object. // Parameters: // retryDelay. @@ -259,10 +216,10 @@ func (hc *HealthCheckImpl) broadcast(th *TabletHealth) { // The localCell for this healthcheck // callback. // A function to call when there is a master change. Used to notify vtgate's buffer to stop buffering. -func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string, callback func(health *TabletHealth)) HealthCheck { +func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string, callback func(health *TabletHealth)) *HealthCheck { log.Infof("loading tablets for cells: %v", *CellsToWatch) - hc := &HealthCheckImpl{ + hc := &HealthCheck{ ts: topoServer, cell: localCell, retryDelay: retryDelay, @@ -314,221 +271,10 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur return hc } -// RegisterStats registers the connection counts stats -func (hc *HealthCheckImpl) RegisterStats() { - stats.NewGaugeDurationFunc( - "TopologyWatcherMaxRefreshLag", - "maximum time since the topology watcher refreshed a cell", - hc.topologyWatcherMaxRefreshLag, - ) - - stats.NewGaugeFunc( - "TopologyWatcherChecksum", - "crc32 checksum of the topology watcher state", - hc.topologyWatcherChecksum, - ) - - stats.NewGaugesFuncWithMultiLabels( - "HealthcheckConnections", - "the number of healthcheck connections registered", - []string{"Keyspace", "ShardName", "TabletType"}, - hc.servingConnStats) - - stats.NewGaugeFunc( - "HealthcheckChecksum", - "crc32 checksum of the current healthcheck state", - hc.stateChecksum) -} - -// ServeHTTP is part of the http.Handler interface. It renders the current state of the discovery gateway tablet cache into json. -func (hc *HealthCheckImpl) ServeHTTP(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - status := hc.CacheStatus() - b, err := json.MarshalIndent(status, "", " ") - if err != nil { - w.Write([]byte(err.Error())) - return - } - - buf := bytes.NewBuffer(nil) - json.HTMLEscape(buf, b) - w.Write(buf.Bytes()) -} - -// servingConnStats returns the number of serving tablets per keyspace/shard/tablet type. -func (hc *HealthCheckImpl) servingConnStats() map[string]int64 { - res := make(map[string]int64) - hc.mu.Lock() - defer hc.mu.Unlock() - for _, th := range hc.healthByAlias { - th.mu.Lock() - if !th.Serving || th.LastError != nil { - th.mu.Unlock() - continue - } - key := fmt.Sprintf("%s.%s.%s", th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)) - th.mu.Unlock() - res[key]++ - } - return res -} - -// stateChecksum returns a crc32 checksum of the healthcheck state -func (hc *HealthCheckImpl) stateChecksum() int64 { - // CacheStatus is sorted so this should be stable across vtgates - cacheStatus := hc.CacheStatus() - var buf bytes.Buffer - for _, st := range cacheStatus { - fmt.Fprintf(&buf, - "%v%v%v%v\n", - st.Cell, - st.Target.Keyspace, - st.Target.Shard, - st.Target.TabletType.String(), - ) - sort.Sort(st.TabletsStats) - for _, ts := range st.TabletsStats { - fmt.Fprintf(&buf, "%v%v\n", ts.Serving, ts.MasterTermStartTime) - } - } - - return int64(crc32.ChecksumIEEE(buf.Bytes())) -} - -// finalizeConn closes the health checking connection and sends the final -// notification about the tablet to downstream. To be called only on exit from -// checkConn(). -func (hc *HealthCheckImpl) finalizeConn(th *tabletHealthCheck) { - th.mu.Lock() - defer th.mu.Unlock() - th.setServingState(false, "finalizeConn closing connection") - // Note: checkConn() exits only when th.ctx.Done() is closed. Thus it's - // safe to simply get Err() value here and assign to LastError. - th.LastError = th.ctx.Err() - if th.Conn != nil { - // Don't use th.ctx because it's already closed. - // Use a separate context, and add a timeout to prevent unbounded waits. - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - th.Conn.Close(ctx) - th.Conn = nil - } -} - -// checkConn performs health checking on the given tablet. -func (hc *HealthCheckImpl) checkConn(th *tabletHealthCheck) { - defer hc.connsWG.Done() - defer hc.finalizeConn(th) - - retryDelay := hc.retryDelay - for { - streamCtx, streamCancel := context.WithCancel(th.ctx) - - // Setup a watcher that restarts the timer every time an update is received. - // If a timeout occurs for a serving tablet, we make it non-serving and send - // a status update. The stream is also terminated so it can be retried. - // servingStatus feeds into the serving var, which keeps track of the serving - // status transmitted by the tablet. - servingStatus := make(chan bool, 1) - // timedout is accessed atomically because there could be a race - // between the goroutine that sets it and the check for its value - // later. - timedout := sync2.NewAtomicBool(false) - go func() { - for { - select { - case <-servingStatus: - continue - case <-time.After(hc.healthCheckTimeout): - timedout.Set(true) - streamCancel() - return - case <-streamCtx.Done(): - // If the stream is done, stop watching. - return - } - } - }() - - // Read stream health responses. - err := th.stream(streamCtx, func(shr *query.StreamHealthResponse) error { - // We received a message. Reset the back-off. - retryDelay = hc.retryDelay - // Don't block on send to avoid deadlocks. - select { - case servingStatus <- shr.Serving: - default: - } - return th.processResponse(hc, shr) - }) - - // streamCancel to make sure the watcher goroutine terminates. - streamCancel() - - if err != nil { - if strings.Contains(err.Error(), "health stats mismatch") { - hc.deleteConn(th.Tablet) - return - } - res := th.SimpleCopy() - hc.broadcast(res) - } - // If there was a timeout send an error. We do this after stream has returned. - // This will ensure that this update prevails over any previous message that - // stream could have sent. - if timedout.Get() { - th.mu.Lock() - th.LastError = fmt.Errorf("healthcheck timed out (latest %v)", th.lastResponseTimestamp) - th.setServingState(false, th.LastError.Error()) - hcErrorCounters.Add([]string{th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)}, 1) - res := th.simpleCopyLocked() - th.mu.Unlock() - hc.broadcast(res) - } - - // Streaming RPC failed e.g. because vttablet was restarted or took too long. - // Sleep until the next retry is up or the context is done/canceled. - select { - case <-th.ctx.Done(): - return - case <-time.After(retryDelay): - // Exponentially back-off to prevent tight-loop. - retryDelay *= 2 - // Limit the retry delay backoff to the health check timeout - if retryDelay > hc.healthCheckTimeout { - retryDelay = hc.healthCheckTimeout - } - } - } -} - -func (hc *HealthCheckImpl) deleteConn(tablet *topodata.Tablet) { - hc.mu.Lock() - defer hc.mu.Unlock() - - key := hc.keyFromTablet(tablet) - tabletAlias := topoproto.TabletAliasString(tablet.Alias) - // delete from authoritative map - th, ok := hc.healthByAlias[tabletAlias] - if !ok { - log.Infof("We have no health data for tablet: %v, it might have been deleted already", tabletAlias) - return - } - th.deleteConnLocked() - delete(hc.healthByAlias, tabletAlias) - // delete from map by keyspace.shard.tabletType - ths, ok := hc.healthData[key] - if !ok { - log.Warningf("We have no health data for target: %v", key) - return - } - delete(ths, tabletAlias) -} - // AddTablet adds the tablet, and starts health check. // It does not block on making connection. // name is an optional tag for the tablet, e.g. an alternative address. -func (hc *HealthCheckImpl) AddTablet(tablet *topodata.Tablet) { +func (hc *HealthCheck) AddTablet(tablet *topodata.Tablet) { log.Infof("Calling AddTablet for tablet: %v", tablet) // check whether we should really add this tablet if !hc.isIncluded(tablet) { @@ -575,26 +321,154 @@ func (hc *HealthCheckImpl) AddTablet(tablet *topodata.Tablet) { hc.broadcast(res) hc.connsWG.Add(1) hc.mu.Unlock() - go hc.checkConn(th) + go th.checkConn(hc) } // RemoveTablet removes the tablet, and stops the health check. // It does not block. -func (hc *HealthCheckImpl) RemoveTablet(tablet *topodata.Tablet) { +func (hc *HealthCheck) RemoveTablet(tablet *topodata.Tablet) { if !hc.isIncluded(tablet) { return } - hc.deleteConn(tablet) + hc.deleteTablet(tablet) } // ReplaceTablet removes the old tablet and adds the new tablet. -func (hc *HealthCheckImpl) ReplaceTablet(old, new *topodata.Tablet) { - hc.deleteConn(old) +func (hc *HealthCheck) ReplaceTablet(old, new *topodata.Tablet) { + hc.deleteTablet(old) hc.AddTablet(new) } +func (hc *HealthCheck) deleteTablet(tablet *topodata.Tablet) { + hc.mu.Lock() + defer hc.mu.Unlock() + + key := hc.keyFromTablet(tablet) + tabletAlias := topoproto.TabletAliasString(tablet.Alias) + // delete from authoritative map + th, ok := hc.healthByAlias[tabletAlias] + if !ok { + log.Infof("We have no health data for tablet: %v, it might have been deleted already", tabletAlias) + return + } + // calling this will end the context associated with th.checkConn + // which will call finalizeConn, which will close the connection + th.cancelFunc() + delete(hc.healthByAlias, tabletAlias) + // delete from map by keyspace.shard.tabletType + ths, ok := hc.healthData[key] + if !ok { + log.Warningf("We have no health data for target: %v", key) + return + } + delete(ths, tabletAlias) +} + +func (hc *HealthCheck) updateHealth(th *tabletHealthCheck, shr *query.StreamHealthResponse, currentTarget *query.Target, trivialNonMasterUpdate bool, isMasterUpdate bool, isMasterChange bool) { + // hc.healthByAlias is authoritative, it should be updated + hc.mu.Lock() + defer hc.mu.Unlock() + + tabletAlias := topoproto.TabletAliasString(shr.TabletAlias) + // this will only change the first time, but it's easiest to set it always rather than check and set + hc.healthByAlias[tabletAlias] = th + + hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) + targetChanged := currentTarget.TabletType != shr.Target.TabletType || currentTarget.Keyspace != shr.Target.Keyspace || currentTarget.Shard != shr.Target.Shard + if targetChanged { + // keyspace and shard are not expected to change, but just in case ... + // move this tabletHealthCheck to the correct map + oldTargetKey := hc.keyFromTarget(currentTarget) + newTargetKey := hc.keyFromTarget(shr.Target) + delete(hc.healthData[oldTargetKey], tabletAlias) + _, ok := hc.healthData[newTargetKey] + if !ok { + hc.healthData[newTargetKey] = make(map[string]*tabletHealthCheck) + } + hc.healthData[newTargetKey][tabletAlias] = th + } + + targetKey := hc.keyFromTarget(shr.Target) + if isMasterUpdate { + if len(hc.healthy[targetKey]) == 0 { + hc.healthy[targetKey] = append(hc.healthy[targetKey], th) + } else { + // We already have one up server, see if we + // need to replace it. + if shr.TabletExternallyReparentedTimestamp < hc.healthy[targetKey][0].MasterTermStartTime { + log.Warningf("not marking healthy master %s as Up for %s because its MasterTermStartTime is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", + topoproto.TabletAliasString(shr.TabletAlias), + topoproto.KeyspaceShardString(shr.Target.Keyspace, shr.Target.Shard), + topoproto.TabletAliasString(hc.healthy[targetKey][0].Tablet.Alias), + shr.TabletExternallyReparentedTimestamp, + hc.healthy[targetKey][0].MasterTermStartTime) + } else { + // Just replace it. + hc.healthy[targetKey][0] = th + } + } + } + if !trivialNonMasterUpdate { + if shr.Target.TabletType != topodata.TabletType_MASTER { + all := hc.healthData[targetKey] + allArray := make([]*tabletHealthCheck, 0, len(all)) + for _, s := range all { + allArray = append(allArray, s) + } + hc.healthy[targetKey] = FilterStatsByReplicationLag(allArray) + } + if targetChanged && currentTarget.TabletType != topodata.TabletType_MASTER { // also recompute old target's healthy list + oldTargetKey := hc.keyFromTarget(currentTarget) + all := hc.healthData[oldTargetKey] + allArray := make([]*tabletHealthCheck, 0, len(all)) + for _, s := range all { + allArray = append(allArray, s) + } + hc.healthy[oldTargetKey] = FilterStatsByReplicationLag(allArray) + } + } + result := th.SimpleCopy() + if isMasterChange { + if hc.masterCallback != nil { + hc.masterCallback(result) + } + log.Errorf("Adding 1 to MasterPromoted counter for tablet: %v, shr.Tablet: %v, shr.TabletType: %v", currentTarget, topoproto.TabletAliasString(shr.TabletAlias), shr.Target.TabletType) + hcMasterPromotedCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard}, 1) + } + // broadcast to subscribers + hc.broadcast(result) + +} + +// Subscribe adds a listener. Only used for testing right now +func (hc *HealthCheck) Subscribe() chan *TabletHealth { + hc.subMu.Lock() + defer hc.subMu.Unlock() + c := make(chan *TabletHealth, 2) + hc.subscribers[c] = struct{}{} + return c +} + +// Unsubscribe removes a listener. Only used for testing right now +func (hc *HealthCheck) Unsubscribe(c chan *TabletHealth) { + hc.subMu.Lock() + defer hc.subMu.Unlock() + delete(hc.subscribers, c) +} + +func (hc *HealthCheck) broadcast(th *TabletHealth) { + hc.subMu.Lock() + defer hc.subMu.Unlock() + for c := range hc.subscribers { + select { + case c <- th: + default: + } + } +} + // CacheStatus returns a displayable version of the cache. -func (hc *HealthCheckImpl) CacheStatus() TabletsCacheStatusList { +func (hc *HealthCheck) CacheStatus() TabletsCacheStatusList { tcsMap := hc.cacheStatusMap() tcsl := make(TabletsCacheStatusList, 0, len(tcsMap)) for _, tcs := range tcsMap { @@ -604,7 +478,7 @@ func (hc *HealthCheckImpl) CacheStatus() TabletsCacheStatusList { return tcsl } -func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { +func (hc *HealthCheck) cacheStatusMap() map[string]*TabletsCacheStatus { tcsMap := make(map[string]*TabletsCacheStatus) hc.mu.Lock() defer hc.mu.Unlock() @@ -625,7 +499,7 @@ func (hc *HealthCheckImpl) cacheStatusMap() map[string]*TabletsCacheStatus { } // Close stops the healthcheck. -func (hc *HealthCheckImpl) Close() error { +func (hc *HealthCheck) Close() error { hc.mu.Lock() for _, th := range hc.healthByAlias { th.cancelFunc() @@ -650,35 +524,13 @@ func (hc *HealthCheckImpl) Close() error { return nil } -// topologyWatcherMaxRefreshLag returns the maximum lag since the watched -// cells were refreshed from the topo server -func (hc *HealthCheckImpl) topologyWatcherMaxRefreshLag() time.Duration { - var lag time.Duration - for _, tw := range hc.topoWatchers { - cellLag := tw.RefreshLag() - if cellLag > lag { - lag = cellLag - } - } - return lag -} - -// topologyWatcherChecksum returns a checksum of the topology watcher state -func (hc *HealthCheckImpl) topologyWatcherChecksum() int64 { - var checksum int64 - for _, tw := range hc.topoWatchers { - checksum = checksum ^ int64(tw.TopoChecksum()) - } - return checksum -} - -// GetHealthyTabletStats returns only the healthy targets. +// GetHealthyTabletStats returns only the healthy tablets. // The returned array is owned by the caller. // For TabletType_MASTER, this will only return at most one entry, // the most recent tablet of type master. // This returns a copy of the data so that callers can access without // synchronization -func (hc *HealthCheckImpl) GetHealthyTabletStats(target *query.Target) []*TabletHealth { +func (hc *HealthCheck) GetHealthyTabletStats(target *query.Target) []*TabletHealth { var result []*TabletHealth hc.mu.Lock() defer hc.mu.Unlock() @@ -688,11 +540,11 @@ func (hc *HealthCheckImpl) GetHealthyTabletStats(target *query.Target) []*Tablet return result } -// GetHealthyTabletStats returns only the healthy targets. +// getTabletStats returns all tablets for the given target. // The returned array is owned by the caller. // For TabletType_MASTER, this will only return at most one entry, // the most recent tablet of type master. -func (hc *HealthCheckImpl) getTabletStats(target *query.Target) []*TabletHealth { +func (hc *HealthCheck) getTabletStats(target *query.Target) []*TabletHealth { var result []*TabletHealth hc.mu.Lock() defer hc.mu.Unlock() @@ -706,7 +558,7 @@ func (hc *HealthCheckImpl) getTabletStats(target *query.Target) []*TabletHealth // WaitForTablets waits for at least one tablet in the given // keyspace / shard / tablet type before returning. The tablets do not // have to be healthy. It will return ctx.Err() if the context is canceled. -func (hc *HealthCheckImpl) WaitForTablets(ctx context.Context, keyspace, shard string, tabletType topodata.TabletType) error { +func (hc *HealthCheck) WaitForTablets(ctx context.Context, keyspace, shard string, tabletType topodata.TabletType) error { targets := []*query.Target{ { Keyspace: keyspace, @@ -721,12 +573,12 @@ func (hc *HealthCheckImpl) WaitForTablets(ctx context.Context, keyspace, shard s // each given target before returning. // It will return ctx.Err() if the context is canceled. // It will return an error if it can't read the necessary topology records. -func (hc *HealthCheckImpl) WaitForAllServingTablets(ctx context.Context, targets []*query.Target) error { +func (hc *HealthCheck) WaitForAllServingTablets(ctx context.Context, targets []*query.Target) error { return hc.waitForTablets(ctx, targets, true) } // waitForTablets is the internal method that polls for tablets. -func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*query.Target, requireServing bool) error { +func (hc *HealthCheck) waitForTablets(ctx context.Context, targets []*query.Target, requireServing bool) error { for { // We nil targets as we find them. allPresent := true @@ -766,15 +618,15 @@ func (hc *HealthCheckImpl) waitForTablets(ctx context.Context, targets []*query. // Target includes cell which we ignore here // because tabletStatsCache is intended to be per-cell -func (hc *HealthCheckImpl) keyFromTarget(target *query.Target) string { +func (hc *HealthCheck) keyFromTarget(target *query.Target) string { return fmt.Sprintf("%s.%s.%d", target.Keyspace, target.Shard, target.TabletType) } -func (hc *HealthCheckImpl) keyFromTablet(tablet *topodata.Tablet) string { +func (hc *HealthCheck) keyFromTablet(tablet *topodata.Tablet) string { return fmt.Sprintf("%s.%s.%d", tablet.Keyspace, tablet.Shard, tablet.Type) } -func (hc *HealthCheckImpl) getAliasByCell(cell string) string { +func (hc *HealthCheck) getAliasByCell(cell string) string { hc.mu.Lock() defer hc.mu.Unlock() @@ -790,7 +642,7 @@ func (hc *HealthCheckImpl) getAliasByCell(cell string) string { return alias } -func (hc *HealthCheckImpl) isIncluded(tablet *topodata.Tablet) bool { +func (hc *HealthCheck) isIncluded(tablet *topodata.Tablet) bool { if tablet.Type == topodata.TabletType_MASTER { return true } @@ -802,3 +654,106 @@ func (hc *HealthCheckImpl) isIncluded(tablet *topodata.Tablet) bool { } return false } + +// topologyWatcherMaxRefreshLag returns the maximum lag since the watched +// cells were refreshed from the topo server +func (hc *HealthCheck) topologyWatcherMaxRefreshLag() time.Duration { + var lag time.Duration + for _, tw := range hc.topoWatchers { + cellLag := tw.RefreshLag() + if cellLag > lag { + lag = cellLag + } + } + return lag +} + +// topologyWatcherChecksum returns a checksum of the topology watcher state +func (hc *HealthCheck) topologyWatcherChecksum() int64 { + var checksum int64 + for _, tw := range hc.topoWatchers { + checksum = checksum ^ int64(tw.TopoChecksum()) + } + return checksum +} + +// RegisterStats registers the connection counts stats +func (hc *HealthCheck) RegisterStats() { + stats.NewGaugeDurationFunc( + "TopologyWatcherMaxRefreshLag", + "maximum time since the topology watcher refreshed a cell", + hc.topologyWatcherMaxRefreshLag, + ) + + stats.NewGaugeFunc( + "TopologyWatcherChecksum", + "crc32 checksum of the topology watcher state", + hc.topologyWatcherChecksum, + ) + + stats.NewGaugesFuncWithMultiLabels( + "HealthcheckConnections", + "the number of healthcheck connections registered", + []string{"Keyspace", "ShardName", "TabletType"}, + hc.servingConnStats) + + stats.NewGaugeFunc( + "HealthcheckChecksum", + "crc32 checksum of the current healthcheck state", + hc.stateChecksum) +} + +// ServeHTTP is part of the http.Handler interface. It renders the current state of the discovery gateway tablet cache into json. +func (hc *HealthCheck) ServeHTTP(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + status := hc.CacheStatus() + b, err := json.MarshalIndent(status, "", " ") + if err != nil { + w.Write([]byte(err.Error())) + return + } + + buf := bytes.NewBuffer(nil) + json.HTMLEscape(buf, b) + w.Write(buf.Bytes()) +} + +// servingConnStats returns the number of serving tablets per keyspace/shard/tablet type. +func (hc *HealthCheck) servingConnStats() map[string]int64 { + res := make(map[string]int64) + hc.mu.Lock() + defer hc.mu.Unlock() + for _, th := range hc.healthByAlias { + th.mu.Lock() + if !th.Serving || th.LastError != nil { + th.mu.Unlock() + continue + } + key := fmt.Sprintf("%s.%s.%s", th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)) + th.mu.Unlock() + res[key]++ + } + return res +} + +// stateChecksum returns a crc32 checksum of the healthcheck state +func (hc *HealthCheck) stateChecksum() int64 { + // CacheStatus is sorted so this should be stable across vtgates + cacheStatus := hc.CacheStatus() + var buf bytes.Buffer + for _, st := range cacheStatus { + fmt.Fprintf(&buf, + "%v%v%v%v\n", + st.Cell, + st.Target.Keyspace, + st.Target.Shard, + st.Target.TabletType.String(), + ) + sort.Sort(st.TabletsStats) + for _, ts := range st.TabletsStats { + fmt.Fprintf(&buf, "%v%v\n", ts.Serving, ts.MasterTermStartTime) + } + } + + return int64(crc32.ChecksumIEEE(buf.Bytes())) +} diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go index 91dc105f6f1..b54b6dd7f60 100644 --- a/go/vt/discovery/healthcheck_test.go +++ b/go/vt/discovery/healthcheck_test.go @@ -81,6 +81,7 @@ func TestHealthCheck(t *testing.T) { assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) shr := &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, @@ -116,6 +117,7 @@ func TestHealthCheck(t *testing.T) { // TabletType changed, should get both old and new event shr = &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, @@ -144,6 +146,7 @@ func TestHealthCheck(t *testing.T) { // Serving & RealtimeStats changed shr = &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: false, TabletExternallyReparentedTimestamp: 0, @@ -163,6 +166,7 @@ func TestHealthCheck(t *testing.T) { // HealthError shr = &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, @@ -182,7 +186,7 @@ func TestHealthCheck(t *testing.T) { testChecksum(t, 1027934207, hc.stateChecksum()) // unchanged // remove tablet - hc.deleteConn(tablet) + hc.deleteTablet(tablet) testChecksum(t, 0, hc.stateChecksum()) } @@ -211,6 +215,7 @@ func TestHealthCheckStreamError(t *testing.T) { // one tablet after receiving a StreamHealthResponse shr := &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, @@ -308,6 +313,7 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { // one tablet after receiving a StreamHealthResponse shr := &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, @@ -368,6 +374,7 @@ func TestHealthCheckTimeout(t *testing.T) { // one tablet after receiving a StreamHealthResponse shr := &querypb.StreamHealthResponse{ + TabletAlias: tablet.Alias, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, @@ -768,8 +775,8 @@ func tabletDialer(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (quer return nil, fmt.Errorf("tablet %v not found", key) } -func createTestHc(ts *topo.Server) *HealthCheckImpl { - return NewHealthCheck(context.Background(), 1*time.Millisecond, time.Hour, ts, "cell", nil).(*HealthCheckImpl) +func createTestHc(ts *topo.Server) *HealthCheck { + return NewHealthCheck(context.Background(), 1*time.Millisecond, time.Hour, ts, "cell", nil) } type fakeConn struct { diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index c011bd66acc..2c35512ac7c 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -8,6 +8,8 @@ import ( "sync" "time" + "vitess.io/vitess/go/sync2" + "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/proto/vtrpc" @@ -23,39 +25,6 @@ import ( "vitess.io/vitess/go/vt/proto/topodata" ) -// tabletHealthCheck maintains the health status of a tablet. A map of this -// structure is maintained in HealthCheckImpl. -type tabletHealthCheck struct { - ctx context.Context - // cancelFunc must be called before discarding tabletHealthCheck. - // This will ensure that the associated checkConn goroutine will terminate. - cancelFunc context.CancelFunc - // Tablet is the tablet object that was sent to HealthCheck.AddTablet. - Tablet *topodata.Tablet - mu sync.Mutex - // Conn is the connection associated with the tablet. - Conn queryservice.QueryService - // Target is the current target as returned by the streaming - // StreamHealth RPC. - Target *query.Target - // Serving describes if the tablet can be serving traffic. - Serving bool - // MasterTermStartTime is the last time at which - // this tablet was either elected the master, or received - // a TabletExternallyReparented event. It is set to 0 if the - // tablet doesn't think it's a master. - MasterTermStartTime int64 - // Stats is the current health status, as received by the - // StreamHealth RPC (replication lag, ...). - Stats *query.RealtimeStats - // LastError is the error we last saw when trying to get the - // tablet's healthcheck. - LastError error - // possibly delete both these - loggedServingState bool - lastResponseTimestamp time.Time // timestamp of the last healthcheck response -} - // TabletHealth represents simple tablet health data that is returned to users of healthcheck. // No synchronization is required because we always return a copy. type TabletHealth struct { @@ -120,6 +89,39 @@ func (th *TabletHealth) getTabletDebugURL() string { return buffer.String() } +// tabletHealthCheck maintains the health status of a tablet. A map of this +// structure is maintained in HealthCheck. +type tabletHealthCheck struct { + ctx context.Context + // cancelFunc must be called before discarding tabletHealthCheck. + // This will ensure that the associated checkConn goroutine will terminate. + cancelFunc context.CancelFunc + // Tablet is the tablet object that was sent to HealthCheck.AddTablet. + Tablet *topodata.Tablet + mu sync.Mutex + // Conn is the connection associated with the tablet. + Conn queryservice.QueryService + // Target is the current target as returned by the streaming + // StreamHealth RPC. + Target *query.Target + // Serving describes if the tablet can be serving traffic. + Serving bool + // MasterTermStartTime is the last time at which + // this tablet was either elected the master, or received + // a TabletExternallyReparented event. It is set to 0 if the + // tablet doesn't think it's a master. + MasterTermStartTime int64 + // Stats is the current health status, as received by the + // StreamHealth RPC (replication lag, ...). + Stats *query.RealtimeStats + // LastError is the error we last saw when trying to get the + // tablet's healthcheck. + LastError error + // possibly delete both these + loggedServingState bool + lastResponseTimestamp time.Time // timestamp of the last healthcheck response +} + // String is defined because we want to print a []*tabletHealthCheck array nicely. func (th *tabletHealthCheck) String() string { th.mu.Lock() @@ -161,13 +163,6 @@ func (th *tabletHealthCheck) DeepEqual(other *tabletHealthCheck) bool { (th.LastError != nil && other.LastError != nil && th.LastError.Error() == other.LastError.Error())) } -func (th *tabletHealthCheck) deleteConnLocked() { - th.mu.Lock() - th.Conn = nil - th.mu.Unlock() - th.cancelFunc() -} - // setServingState sets the tablet state to the given value. // // If the state changes, it logs the change so that failures @@ -222,18 +217,8 @@ func (th *tabletHealthCheck) getConnection() queryservice.QueryService { return th.Conn } -func (th *tabletHealthCheck) closeConnection(ctx context.Context, err error) { - th.mu.Lock() - defer th.mu.Unlock() - log.Warningf("tablet %v healthcheck stream error: %v", th.Tablet.Alias, err) - th.setServingState(false, err.Error()) - th.LastError = err - _ = th.Conn.Close(ctx) - th.Conn = nil -} - // processResponse reads one health check response, and updates health -func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.StreamHealthResponse) error { +func (th *tabletHealthCheck) processResponse(hc *HealthCheck, shr *query.StreamHealthResponse) error { select { case <-th.ctx.Done(): return th.ctx.Err() @@ -267,39 +252,7 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str trivialNonMasterUpdate := th.LastError == nil && th.Serving && shr.RealtimeStats.HealthError == "" && shr.Serving && currentTarget.TabletType != topodata.TabletType_MASTER && currentTarget.TabletType == shr.Target.TabletType isMasterUpdate := shr.Target.TabletType == topodata.TabletType_MASTER - // Track how often a tablet gets promoted to master. It is used for - // comparing against the variables in go/vtgate/buffer/variables.go. - isMasterChange := currentTarget.TabletType != topodata.TabletType_MASTER && shr.Target.TabletType == topodata.TabletType_MASTER - th.mu.Unlock() - - // hc.healthByAlias is authoritative, it should be updated - hc.mu.Lock() - tabletAlias := topoproto.TabletAliasString(th.Tablet.Alias) - // this will only change the first time, but it's easiest to set it always rather than check and set - hc.healthByAlias[tabletAlias] = th - hc.mu.Unlock() - - hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) - targetChanged := currentTarget.TabletType != shr.Target.TabletType || currentTarget.Keyspace != shr.Target.Keyspace || currentTarget.Shard != shr.Target.Shard - if targetChanged { - // keyspace and shard are not expected to change, but just in case ... - // move this tabletHealthCheck to the correct map - oldTargetKey := hc.keyFromTarget(currentTarget) - newTargetKey := hc.keyFromTarget(shr.Target) - tabletAlias := topoproto.TabletAliasString(shr.TabletAlias) - hc.mu.Lock() - delete(hc.healthData[oldTargetKey], tabletAlias) - _, ok := hc.healthData[newTargetKey] - if !ok { - hc.healthData[newTargetKey] = make(map[string]*tabletHealthCheck) - } - hc.healthData[newTargetKey][tabletAlias] = th - hc.mu.Unlock() - } - - // Update our record - th.mu.Lock() - defer th.mu.Unlock() + isMasterChange := th.Target.TabletType != topodata.TabletType_MASTER && shr.Target.TabletType == topodata.TabletType_MASTER th.lastResponseTimestamp = time.Now() th.Target = shr.Target th.MasterTermStartTime = shr.TabletExternallyReparentedTimestamp @@ -310,59 +263,124 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheckImpl, shr *query.Str reason = "healthCheck update error: " + healthErr.Error() } th.setServingState(serving, reason) + th.mu.Unlock() + // notify downstream for master change + hc.updateHealth(th, shr, currentTarget, trivialNonMasterUpdate, isMasterUpdate, isMasterChange) + return nil +} + +// checkConn performs health checking on the given tablet. +func (th *tabletHealthCheck) checkConn(hc *HealthCheck) { + defer hc.connsWG.Done() + defer th.finalizeConn() + + retryDelay := hc.retryDelay + for { + streamCtx, streamCancel := context.WithCancel(th.ctx) - hc.mu.Lock() - defer hc.mu.Unlock() - targetKey := hc.keyFromTarget(shr.Target) - if isMasterUpdate { - if len(hc.healthy[targetKey]) == 0 { - hc.healthy[targetKey] = append(hc.healthy[targetKey], th) - } else { - // We already have one up server, see if we - // need to replace it. - if th.MasterTermStartTime < hc.healthy[targetKey][0].MasterTermStartTime { - log.Warningf("not marking healthy master %s as Up for %s because its MasterTermStartTime is smaller than the highest known timestamp from previous MASTERs %s: %d < %d ", - topoproto.TabletAliasString(shr.TabletAlias), - topoproto.KeyspaceShardString(shr.Target.Keyspace, shr.Target.Shard), - topoproto.TabletAliasString(hc.healthy[targetKey][0].Tablet.Alias), - th.MasterTermStartTime, - hc.healthy[targetKey][0].MasterTermStartTime) - } else { - // Just replace it. - hc.healthy[targetKey][0] = th + // Setup a watcher that restarts the timer every time an update is received. + // If a timeout occurs for a serving tablet, we make it non-serving and send + // a status update. The stream is also terminated so it can be retried. + // servingStatus feeds into the serving var, which keeps track of the serving + // status transmitted by the tablet. + servingStatus := make(chan bool, 1) + // timedout is accessed atomically because there could be a race + // between the goroutine that sets it and the check for its value + // later. + timedout := sync2.NewAtomicBool(false) + go func() { + for { + select { + case <-servingStatus: + continue + case <-time.After(hc.healthCheckTimeout): + timedout.Set(true) + streamCancel() + return + case <-streamCtx.Done(): + // If the stream is done, stop watching. + return + } } - } - } - if !trivialNonMasterUpdate { - if shr.Target.TabletType != topodata.TabletType_MASTER { - all := hc.healthData[targetKey] - allArray := make([]*tabletHealthCheck, 0, len(all)) - for _, s := range all { - allArray = append(allArray, s) + }() + + // Read stream health responses. + err := th.stream(streamCtx, func(shr *query.StreamHealthResponse) error { + // We received a message. Reset the back-off. + retryDelay = hc.retryDelay + // Don't block on send to avoid deadlocks. + select { + case servingStatus <- shr.Serving: + default: } - hc.healthy[targetKey] = FilterStatsByReplicationLag(allArray) + return th.processResponse(hc, shr) + }) + + // streamCancel to make sure the watcher goroutine terminates. + streamCancel() + + if err != nil { + if strings.Contains(err.Error(), "health stats mismatch") { + hc.deleteTablet(th.Tablet) + return + } + res := th.SimpleCopy() + hc.broadcast(res) + } + // If there was a timeout send an error. We do this after stream has returned. + // This will ensure that this update prevails over any previous message that + // stream could have sent. + if timedout.Get() { + th.mu.Lock() + th.LastError = fmt.Errorf("healthcheck timed out (latest %v)", th.lastResponseTimestamp) + th.setServingState(false, th.LastError.Error()) + hcErrorCounters.Add([]string{th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)}, 1) + res := th.simpleCopyLocked() + th.mu.Unlock() + hc.broadcast(res) } - if targetChanged && currentTarget.TabletType != topodata.TabletType_MASTER { // also recompute old target's healthy list - oldTargetKey := hc.keyFromTarget(currentTarget) - all := hc.healthData[oldTargetKey] - allArray := make([]*tabletHealthCheck, 0, len(all)) - for _, s := range all { - allArray = append(allArray, s) + + // Streaming RPC failed e.g. because vttablet was restarted or took too long. + // Sleep until the next retry is up or the context is done/canceled. + select { + case <-th.ctx.Done(): + return + case <-time.After(retryDelay): + // Exponentially back-off to prevent tight-loop. + retryDelay *= 2 + // Limit the retry delay backoff to the health check timeout + if retryDelay > hc.healthCheckTimeout { + retryDelay = hc.healthCheckTimeout } - hc.healthy[oldTargetKey] = FilterStatsByReplicationLag(allArray) } } +} - // notify downstream for master change - result := th.simpleCopyLocked() - if isMasterChange { - if hc.masterCallback != nil { - hc.masterCallback(result) - } - log.Errorf("Adding 1 to MasterPromoted counter for tablet: %v, shr.Tablet: %v, shr.TabletType: %v", currentTarget, topoproto.TabletAliasString(shr.TabletAlias), shr.Target.TabletType) - hcMasterPromotedCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard}, 1) +func (th *tabletHealthCheck) closeConnection(ctx context.Context, err error) { + th.mu.Lock() + defer th.mu.Unlock() + log.Warningf("tablet %v healthcheck stream error: %v", th.Tablet.Alias, err) + th.setServingState(false, err.Error()) + th.LastError = err + _ = th.Conn.Close(ctx) + th.Conn = nil +} + +// finalizeConn closes the health checking connection. +// To be called only on exit from checkConn(). +func (th *tabletHealthCheck) finalizeConn() { + th.mu.Lock() + defer th.mu.Unlock() + th.setServingState(false, "finalizeConn closing connection") + // Note: checkConn() exits only when th.ctx.Done() is closed. Thus it's + // safe to simply get Err() value here and assign to LastError. + th.LastError = th.ctx.Err() + if th.Conn != nil { + // Don't use th.ctx because it's already closed. + // Use a separate context, and add a timeout to prevent unbounded waits. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = th.Conn.Close(ctx) + th.Conn = nil } - // broadcast to subscribers - hc.broadcast(result) - return nil } diff --git a/go/vt/vtgate/api.go b/go/vt/vtgate/api.go index 46b25ce301b..bf4d1ca5ef5 100644 --- a/go/vt/vtgate/api.go +++ b/go/vt/vtgate/api.go @@ -88,7 +88,7 @@ func getItemPath(url string) string { return parts[1] } -func initAPI(ctx context.Context, hc discovery.HealthCheck) { +func initAPI(ctx context.Context, hc *discovery.HealthCheck) { // Healthcheck real time status per (cell, keyspace, tablet type, metric). handleCollection("health-check", func(r *http.Request) (interface{}, error) { cacheStatus := hc.CacheStatus() diff --git a/go/vt/vtgate/discoverygateway.go b/go/vt/vtgate/discoverygateway.go index a8ca17125e5..057e620d7f2 100644 --- a/go/vt/vtgate/discoverygateway.go +++ b/go/vt/vtgate/discoverygateway.go @@ -411,6 +411,6 @@ func NewShardError(in error, target *querypb.Target, tablet *topodatapb.Tablet) // HealthCheck should never be called on a DiscoveryGateway // This exists only to satisfy the interface -func (dg *DiscoveryGateway) HealthCheck() discovery.HealthCheck { +func (dg *DiscoveryGateway) HealthCheck() *discovery.HealthCheck { return nil } diff --git a/go/vt/vtgate/gateway.go b/go/vt/vtgate/gateway.go index ed7b56b1057..efeef6b87a4 100644 --- a/go/vt/vtgate/gateway.go +++ b/go/vt/vtgate/gateway.go @@ -64,7 +64,7 @@ type Gateway interface { CacheStatus() TabletCacheStatusList // HealthCheck returns a reference to the healthCheck being used by this gateway - HealthCheck() discovery.HealthCheck + HealthCheck() *discovery.HealthCheck } // Creator is the factory method which can create the actual gateway object. diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index dca926b8904..622c4d9689a 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -52,7 +52,7 @@ func init() { // This implementation uses the new healthcheck module. type TabletGateway struct { queryservice.QueryService - hc discovery.HealthCheck + hc *discovery.HealthCheck srvTopoServer srvtopo.Server localCell string retryCount int @@ -309,6 +309,6 @@ func (gw *TabletGateway) nextTablet(cell string, tablets []*discovery.TabletHeal } // HealthCheck satisfies the Gateway interface -func (gw *TabletGateway) HealthCheck() discovery.HealthCheck { +func (gw *TabletGateway) HealthCheck() *discovery.HealthCheck { return gw.hc } From 08deeac18d1271b235bf094627910a91274c13f8 Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 21 May 2020 21:34:34 -0700 Subject: [PATCH 31/39] healthcheck: use channel to notify buffer of new master Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 9 +-------- go/vt/discovery/healthcheck_test.go | 2 +- go/vt/discovery/legacy_healthcheck.go | 2 +- go/vt/vtgate/buffer/buffer.go | 7 +++++-- go/vt/vtgate/tabletgateway.go | 22 +++++++++++++++++----- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index ee9c20a6334..2e3f1e0387d 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -190,9 +190,6 @@ type HealthCheck struct { connsWG sync.WaitGroup // topology watchers that inform healthcheck of tablets being added and deleted topoWatchers []*TopologyWatcher - // used to inform vtgate buffer when new master is detected - // TODO: buffer should subscribe to healthcheck instead of setting a callback - masterCallback func(health *TabletHealth) // cellAliases is a cache of cell aliases cellAliases map[string]string // mutex to protect subscribers @@ -216,7 +213,7 @@ type HealthCheck struct { // The localCell for this healthcheck // callback. // A function to call when there is a master change. Used to notify vtgate's buffer to stop buffering. -func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string, callback func(health *TabletHealth)) *HealthCheck { +func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Duration, topoServer *topo.Server, localCell string) *HealthCheck { log.Infof("loading tablets for cells: %v", *CellsToWatch) hc := &HealthCheck{ @@ -224,7 +221,6 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur cell: localCell, retryDelay: retryDelay, healthCheckTimeout: healthCheckTimeout, - masterCallback: callback, healthByAlias: make(map[string]*tabletHealthCheck), healthData: make(map[string]map[string]*tabletHealthCheck), healthy: make(map[string][]*tabletHealthCheck), @@ -429,9 +425,6 @@ func (hc *HealthCheck) updateHealth(th *tabletHealthCheck, shr *query.StreamHeal } result := th.SimpleCopy() if isMasterChange { - if hc.masterCallback != nil { - hc.masterCallback(result) - } log.Errorf("Adding 1 to MasterPromoted counter for tablet: %v, shr.Tablet: %v, shr.TabletType: %v", currentTarget, topoproto.TabletAliasString(shr.TabletAlias), shr.Target.TabletType) hcMasterPromotedCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard}, 1) } diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go index b54b6dd7f60..080b3f07025 100644 --- a/go/vt/discovery/healthcheck_test.go +++ b/go/vt/discovery/healthcheck_test.go @@ -776,7 +776,7 @@ func tabletDialer(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (quer } func createTestHc(ts *topo.Server) *HealthCheck { - return NewHealthCheck(context.Background(), 1*time.Millisecond, time.Hour, ts, "cell", nil) + return NewHealthCheck(context.Background(), 1*time.Millisecond, time.Hour, ts, "cell") } type fakeConn struct { diff --git a/go/vt/discovery/legacy_healthcheck.go b/go/vt/discovery/legacy_healthcheck.go index 7e17734ad8c..c6edb06df55 100644 --- a/go/vt/discovery/legacy_healthcheck.go +++ b/go/vt/discovery/legacy_healthcheck.go @@ -297,7 +297,7 @@ type LegacyHealthCheck interface { // listener before any tablets are added to the healthcheck. SetListener(listener LegacyHealthCheckStatsListener, sendDownEvents bool) // WaitForInitialStatsUpdates waits until all tablets added via - // AddTablet() call were propagated to the listener via corresponding + // AddTablet() call were propagated to the listener via correspondingdiscovert // StatsUpdate() calls. Note that code path from AddTablet() to // corresponding StatsUpdate() is asynchronous but not cancelable, thus // this function is also non-cancelable and can't return error. Also diff --git a/go/vt/vtgate/buffer/buffer.go b/go/vt/vtgate/buffer/buffer.go index 0568d6df950..84ab628cfc9 100644 --- a/go/vt/vtgate/buffer/buffer.go +++ b/go/vt/vtgate/buffer/buffer.go @@ -213,9 +213,12 @@ func (b *Buffer) WaitForFailoverEnd(ctx context.Context, keyspace, shard string, return sb.waitForFailoverEnd(ctx, keyspace, shard, err) } -// NewMasterDetected notifies the buffer to record a new master +// ProcessMasterHealth notifies the buffer to record a new master // and end any failover buffering that may be in progress -func (b *Buffer) NewMasterDetected(th *discovery.TabletHealth) { +func (b *Buffer) ProcessMasterHealth(th *discovery.TabletHealth) { + if th.Target.TabletType != topodatapb.TabletType_MASTER { + panic(fmt.Sprintf("BUG: non MASTER TabletHealth object must not be forwarded: %#v", th)) + } timestamp := th.MasterTermStartTime if timestamp == 0 { // Masters where TabletExternallyReparented was never called will return 0. diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index 622c4d9689a..1c9b9cbeaf1 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -82,10 +82,7 @@ func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string log.Exitf("Unable to create new TabletGateway: %v", err) } } - b := buffer.New() - // callback to notify buffer when to end failover - newMasterDetected := b.NewMasterDetected - hc := discovery.NewHealthCheck(ctx, *HealthCheckRetryDelay, *HealthCheckTimeout, topoServer, localCell, newMasterDetected) + hc := discovery.NewHealthCheck(ctx, *HealthCheckRetryDelay, *HealthCheckTimeout, topoServer, localCell) gw := &TabletGateway{ hc: hc, @@ -93,8 +90,23 @@ func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string localCell: localCell, retryCount: *RetryCount, statusAggregators: make(map[string]*TabletStatusAggregator), - buffer: b, + buffer: buffer.New(), } + // subscribe to healthcheck updates so that buffer can be notified if needed + // we run this in a separate goroutine so that normal processing doesn't need to block + hcChan := hc.Subscribe() + go func(ctx context.Context, c chan *discovery.TabletHealth, buffer *buffer.Buffer) { + for { + select { + case <-ctx.Done(): + return + case result := <-hcChan: + if result.Target.TabletType == topodatapb.TabletType_MASTER { + buffer.ProcessMasterHealth(result) + } + } + } + }(ctx, hcChan, gw.buffer) gw.QueryService = queryservice.Wrap(nil, gw.withRetry) return gw } From edd4066c2a424e8745b07aa53cc4bb94ab8465ba Mon Sep 17 00:00:00 2001 From: deepthi Date: Fri, 22 May 2020 16:13:40 -0700 Subject: [PATCH 32/39] healthcheck: close subscriber goroutine when healthcheck is closed Signed-off-by: deepthi --- go/vt/vtgate/tabletgateway.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index 1c9b9cbeaf1..255c492326a 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -95,18 +95,24 @@ func NewTabletGateway(ctx context.Context, serv srvtopo.Server, localCell string // subscribe to healthcheck updates so that buffer can be notified if needed // we run this in a separate goroutine so that normal processing doesn't need to block hcChan := hc.Subscribe() + bufferCtx, bufferCancel := context.WithCancel(ctx) go func(ctx context.Context, c chan *discovery.TabletHealth, buffer *buffer.Buffer) { for { select { case <-ctx.Done(): return case result := <-hcChan: + if result == nil { + // If result is nil it must mean the channel has been closed. Stop goroutine in that case + bufferCancel() + return + } if result.Target.TabletType == topodatapb.TabletType_MASTER { buffer.ProcessMasterHealth(result) } } } - }(ctx, hcChan, gw.buffer) + }(bufferCtx, hcChan, gw.buffer) gw.QueryService = queryservice.Wrap(nil, gw.withRetry) return gw } From 45b9b1dc281f721b225b0cfe2c05fb2dbe543786 Mon Sep 17 00:00:00 2001 From: deepthi Date: Sat, 23 May 2020 16:52:58 -0700 Subject: [PATCH 33/39] healthcheck: fix endtoend setup to use correct gateway Signed-off-by: deepthi --- go/test/endtoend/tabletgateway/healthcheck/main_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/tabletgateway/healthcheck/main_test.go b/go/test/endtoend/tabletgateway/healthcheck/main_test.go index c3431c41a4b..f4fa9940b28 100644 --- a/go/test/endtoend/tabletgateway/healthcheck/main_test.go +++ b/go/test/endtoend/tabletgateway/healthcheck/main_test.go @@ -64,7 +64,6 @@ func TestMain(m *testing.M) { exitCode := func() int { clusterInstance = cluster.NewCluster(cell, "localhost") - clusterInstance.VtGateExtraArgs = []string{"-gateway_implementation", "tabletgateway"} clusterInstance.VtTabletExtraArgs = []string{"-health_check_interval", "1s"} defer clusterInstance.Teardown() @@ -86,7 +85,11 @@ func TestMain(m *testing.M) { } // Start vtgate - err = clusterInstance.StartVtgate() + vtgateInstance := clusterInstance.GetVtgateInstance() + // ensure it is torn down during cluster TearDown + clusterInstance.VtgateProcess = *vtgateInstance + vtgateInstance.GatewayImplementation = "tabletgateway" + err = vtgateInstance.Setup() if err != nil { return 1 } From 3f172ca06e7b7fd55029b7737f45b41accc4952d Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 25 May 2020 12:16:08 +0200 Subject: [PATCH 34/39] addressed review comments Signed-off-by: Andres Taylor --- .../tabletgateway/buffer/buffer_test.go | 2 +- .../cellalias/cell_alias_test.go | 2 +- .../tabletgateway/healthcheck/main_test.go | 2 +- .../tabletgateway/healthcheck/vtgate_test.go | 2 +- go/vt/discovery/healthcheck.go | 6 +- go/vt/discovery/replicationlag_test.go | 39 ++- go/vt/discovery/tablet_health.go | 325 +----------------- go/vt/discovery/tablet_health_check.go | 324 +++++++++++++++++ 8 files changed, 367 insertions(+), 335 deletions(-) create mode 100644 go/vt/discovery/tablet_health_check.go diff --git a/go/test/endtoend/tabletgateway/buffer/buffer_test.go b/go/test/endtoend/tabletgateway/buffer/buffer_test.go index 1cf1e768c0c..62d98ab641a 100644 --- a/go/test/endtoend/tabletgateway/buffer/buffer_test.go +++ b/go/test/endtoend/tabletgateway/buffer/buffer_test.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Vitess Authors. +Copyright 2020 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/test/endtoend/tabletgateway/cellalias/cell_alias_test.go b/go/test/endtoend/tabletgateway/cellalias/cell_alias_test.go index 089ec7a8666..3ee4809ff8f 100644 --- a/go/test/endtoend/tabletgateway/cellalias/cell_alias_test.go +++ b/go/test/endtoend/tabletgateway/cellalias/cell_alias_test.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Vitess Authors. +Copyright 2020 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/test/endtoend/tabletgateway/healthcheck/main_test.go b/go/test/endtoend/tabletgateway/healthcheck/main_test.go index f4fa9940b28..084586a9b3a 100644 --- a/go/test/endtoend/tabletgateway/healthcheck/main_test.go +++ b/go/test/endtoend/tabletgateway/healthcheck/main_test.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Vitess Authors. +Copyright 2020 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/test/endtoend/tabletgateway/healthcheck/vtgate_test.go b/go/test/endtoend/tabletgateway/healthcheck/vtgate_test.go index cd67abe1978..26925afd96d 100644 --- a/go/test/endtoend/tabletgateway/healthcheck/vtgate_test.go +++ b/go/test/endtoend/tabletgateway/healthcheck/vtgate_test.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Vitess Authors. +Copyright 2020 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 2e3f1e0387d..bfbf86b67c8 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Vitess Authors. +Copyright 2020 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -277,9 +277,9 @@ func (hc *HealthCheck) AddTablet(tablet *topodata.Tablet) { return } hc.mu.Lock() + defer hc.mu.Unlock() if hc.healthByAlias == nil { // already closed. - hc.mu.Unlock() return } ctx, cancelFunc := context.WithCancel(context.Background()) @@ -301,7 +301,6 @@ func (hc *HealthCheck) AddTablet(tablet *topodata.Tablet) { // TODO: can this ever already exist? if _, ok := hc.healthByAlias[tabletAlias]; ok { log.Errorf("Program bug") - hc.mu.Unlock() return } hc.healthByAlias[tabletAlias] = th @@ -316,7 +315,6 @@ func (hc *HealthCheck) AddTablet(tablet *topodata.Tablet) { res := th.SimpleCopy() hc.broadcast(res) hc.connsWG.Add(1) - hc.mu.Unlock() go th.checkConn(hc) } diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index 83e61e77402..c95eeffa1be 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -20,6 +20,8 @@ import ( "fmt" "testing" + "vitess.io/vitess/go/test/utils" + querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/topo" ) @@ -42,12 +44,8 @@ func TestFilterByReplicationLagUnhealthy(t *testing.T) { Stats: &querypb.RealtimeStats{}, } got := FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2}) - if len(got) != 1 { - t.Errorf("len(FilterStatsByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}])) = %v, want 1", len(got)) - } - if len(got) > 0 && !got[0].DeepEqual(ts1) { - t.Errorf("FilterStatsByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}]) = %+v, want %+v", got[0], ts1) - } + want := []*tabletHealthCheck{ts1} + mustMatch(t, want, got, "FilterStatsByReplicationLag") } func TestFilterByReplicationLag(t *testing.T) { @@ -151,9 +149,9 @@ func TestFilterByReplicationLagThreeTabletMin(t *testing.T) { Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } got := FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2, ts3, ts4}) - if len(got) != 3 || !got[0].DeepEqual(ts1) || !got[1].DeepEqual(ts2) || !got[2].DeepEqual(ts3) { - t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) - } + want := []*tabletHealthCheck{ts1, ts2, ts3} + mustMatch(t, want, got, "FilterStatsByReplicationLag") + // lags of (11m, 10m, 1s, 1s) - reordered tablets returns the same 3 items where the slightly delayed one that is returned is the 10m and 11m ones. ts1 = &tabletHealthCheck{ Tablet: topo.NewTablet(1, "cell", "host1"), @@ -176,9 +174,8 @@ func TestFilterByReplicationLagThreeTabletMin(t *testing.T) { Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } got = FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2, ts3, ts4}) - if len(got) != 3 || !got[0].DeepEqual(ts3) || !got[1].DeepEqual(ts4) || !got[2].DeepEqual(ts2) { - t.Errorf("FilterStatsByReplicationLag([1s, 1s, 10m, 11m]) = %+v, want [1s, 1s, 10m]", got) - } + want = []*tabletHealthCheck{ts3, ts4, ts2} + mustMatch(t, want, got, "FilterStatsByReplicationLag") // Reset to the default testSetMinNumTablets(2) } @@ -198,9 +195,9 @@ func TestFilterStatsByReplicationLagOneTabletMin(t *testing.T) { Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } got := FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2}) - if len(got) != 1 || !got[0].DeepEqual(ts1) { - t.Errorf("FilterStatsByReplicationLag([1s, 100m]) = %+v, want [1s]", got) - } + want := []*tabletHealthCheck{ts1} + mustMatch(t, want, got, "FilterStatsByReplicationLag") + // lags of (1m, 100m) - return only healthy tablet if that is all that is healthy enough. ts1 = &tabletHealthCheck{ Tablet: topo.NewTablet(1, "cell", "host1"), @@ -213,9 +210,15 @@ func TestFilterStatsByReplicationLagOneTabletMin(t *testing.T) { Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } got = FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2}) - if len(got) != 1 || !got[0].DeepEqual(ts1) { - t.Errorf("FilterStatsByReplicationLag([1m, 100m]) = %+v, want [1m]", got) - } + want = []*tabletHealthCheck{ts1} + mustMatch(t, want, got, "FilterStatsByReplicationLag") // Reset to the default testSetMinNumTablets(2) } + +var mustMatch = utils.MustMatchFn( + []interface{}{ // types with unexported fields + tabletHealthCheck{}, + }, + []string{".mu"}, // ignored fields +) diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 2c35512ac7c..e3b37e759ff 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -1,23 +1,26 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package discovery import ( "bytes" - "context" - "fmt" "strings" - "sync" - "time" - "vitess.io/vitess/go/sync2" - - "vitess.io/vitess/go/vt/grpcclient" - "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/proto/vtrpc" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/topotools" - "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/queryservice" - "vitess.io/vitess/go/vt/vttablet/tabletconn" "github.com/golang/protobuf/proto" "vitess.io/vitess/go/netutil" @@ -88,299 +91,3 @@ func (th *TabletHealth) getTabletDebugURL() string { tabletURLTemplate.Execute(&buffer, th) return buffer.String() } - -// tabletHealthCheck maintains the health status of a tablet. A map of this -// structure is maintained in HealthCheck. -type tabletHealthCheck struct { - ctx context.Context - // cancelFunc must be called before discarding tabletHealthCheck. - // This will ensure that the associated checkConn goroutine will terminate. - cancelFunc context.CancelFunc - // Tablet is the tablet object that was sent to HealthCheck.AddTablet. - Tablet *topodata.Tablet - mu sync.Mutex - // Conn is the connection associated with the tablet. - Conn queryservice.QueryService - // Target is the current target as returned by the streaming - // StreamHealth RPC. - Target *query.Target - // Serving describes if the tablet can be serving traffic. - Serving bool - // MasterTermStartTime is the last time at which - // this tablet was either elected the master, or received - // a TabletExternallyReparented event. It is set to 0 if the - // tablet doesn't think it's a master. - MasterTermStartTime int64 - // Stats is the current health status, as received by the - // StreamHealth RPC (replication lag, ...). - Stats *query.RealtimeStats - // LastError is the error we last saw when trying to get the - // tablet's healthcheck. - LastError error - // possibly delete both these - loggedServingState bool - lastResponseTimestamp time.Time // timestamp of the last healthcheck response -} - -// String is defined because we want to print a []*tabletHealthCheck array nicely. -func (th *tabletHealthCheck) String() string { - th.mu.Lock() - defer th.mu.Unlock() - return fmt.Sprintf("tabletHealthCheck{Tablet: %v,Target: %v,Serving: %v, MasterTermStartTime: %v, Stats: %v, LastError: %v", - th.Tablet, th.Target, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) -} - -// SimpleCopy returns a TabletHealth with all the necessary fields copied from tabletHealthCheck. -// Note that this is not a deep copy because we point to the same underlying RealtimeStats. -// That is fine because the RealtimeStats object is never changed after creation. -func (th *tabletHealthCheck) SimpleCopy() *TabletHealth { - th.mu.Lock() - defer th.mu.Unlock() - return th.simpleCopyLocked() -} - -func (th *tabletHealthCheck) simpleCopyLocked() *TabletHealth { - return &TabletHealth{ - Conn: th.Conn, - Tablet: th.Tablet, - Target: th.Target, - Stats: th.Stats, - LastError: th.LastError, - MasterTermStartTime: th.MasterTermStartTime, - Serving: th.Serving, - } -} - -// DeepEqual compares two tabletHealthCheck. Since we include protos, we -// need to use proto.Equal on these. -func (th *tabletHealthCheck) DeepEqual(other *tabletHealthCheck) bool { - return proto.Equal(th.Tablet, other.Tablet) && - proto.Equal(th.Target, other.Target) && - th.Serving == other.Serving && - th.MasterTermStartTime == other.MasterTermStartTime && - proto.Equal(th.Stats, other.Stats) && - ((th.LastError == nil && other.LastError == nil) || - (th.LastError != nil && other.LastError != nil && th.LastError.Error() == other.LastError.Error())) -} - -// setServingState sets the tablet state to the given value. -// -// If the state changes, it logs the change so that failures -// from the health check connection are logged the first time, -// but don't continue to log if the connection stays down. -// -// th.mu must be locked before calling this function -func (th *tabletHealthCheck) setServingState(serving bool, reason string) { - if !th.loggedServingState || (serving != th.Serving) { - // Emit the log from a separate goroutine to avoid holding - // the th lock while logging is happening - go log.Infof("HealthCheckUpdate(Serving State): tablet: %v serving => %v for %v/%v (%v) reason: %s", - topotools.TabletIdent(th.Tablet), - serving, - th.Tablet.GetKeyspace(), - th.Tablet.GetShard(), - th.Target.GetTabletType(), - reason, - ) - th.loggedServingState = true - } - th.Serving = serving -} - -// stream streams healthcheck responses to callback. -func (th *tabletHealthCheck) stream(ctx context.Context, callback func(*query.StreamHealthResponse) error) error { - conn := th.getConnection() - if conn == nil { - // This signals the caller to retry - return nil - } - err := conn.StreamHealth(ctx, callback) - if err != nil { - // Depending on the specific error the caller can take action - th.closeConnection(ctx, err) - } - return err -} - -func (th *tabletHealthCheck) getConnection() queryservice.QueryService { - th.mu.Lock() - defer th.mu.Unlock() - if th.Conn == nil { - conn, err := tabletconn.GetDialer()(th.Tablet, grpcclient.FailFast(true)) - if err != nil { - th.LastError = err - return nil - } - th.Conn = conn - th.LastError = nil - } - return th.Conn -} - -// processResponse reads one health check response, and updates health -func (th *tabletHealthCheck) processResponse(hc *HealthCheck, shr *query.StreamHealthResponse) error { - select { - case <-th.ctx.Done(): - return th.ctx.Err() - default: - } - - // Check for invalid data, better than panicking. - if shr.Target == nil || shr.RealtimeStats == nil { - return fmt.Errorf("health stats is not valid: %v", shr) - } - - // an app-level error from tablet, force serving state. - var healthErr error - serving := shr.Serving - if shr.RealtimeStats.HealthError != "" { - healthErr = fmt.Errorf("vttablet error: %v", shr.RealtimeStats.HealthError) - serving = false - } - - if shr.TabletAlias != nil && !proto.Equal(shr.TabletAlias, th.Tablet.Alias) { - // TabletAlias change means that the host:port has been taken over by another tablet - // We could cancel / exit the healthcheck for this tablet right away - // However, we defer it until the next topo refresh informs us of the change because that is - // the only way to discover the new host/port - return vterrors.New(vtrpc.Code_FAILED_PRECONDITION, fmt.Sprintf("health stats mismatch, tablet %+v alias does not match response alias %v", th.Tablet, shr.TabletAlias)) - } - - th.mu.Lock() - currentTarget := th.Target - // check whether this is a trivial update so as to update healthy map - trivialNonMasterUpdate := th.LastError == nil && th.Serving && shr.RealtimeStats.HealthError == "" && shr.Serving && - currentTarget.TabletType != topodata.TabletType_MASTER && currentTarget.TabletType == shr.Target.TabletType - isMasterUpdate := shr.Target.TabletType == topodata.TabletType_MASTER - isMasterChange := th.Target.TabletType != topodata.TabletType_MASTER && shr.Target.TabletType == topodata.TabletType_MASTER - th.lastResponseTimestamp = time.Now() - th.Target = shr.Target - th.MasterTermStartTime = shr.TabletExternallyReparentedTimestamp - th.Stats = shr.RealtimeStats - th.LastError = healthErr - reason := "healthCheck update" - if healthErr != nil { - reason = "healthCheck update error: " + healthErr.Error() - } - th.setServingState(serving, reason) - th.mu.Unlock() - // notify downstream for master change - hc.updateHealth(th, shr, currentTarget, trivialNonMasterUpdate, isMasterUpdate, isMasterChange) - return nil -} - -// checkConn performs health checking on the given tablet. -func (th *tabletHealthCheck) checkConn(hc *HealthCheck) { - defer hc.connsWG.Done() - defer th.finalizeConn() - - retryDelay := hc.retryDelay - for { - streamCtx, streamCancel := context.WithCancel(th.ctx) - - // Setup a watcher that restarts the timer every time an update is received. - // If a timeout occurs for a serving tablet, we make it non-serving and send - // a status update. The stream is also terminated so it can be retried. - // servingStatus feeds into the serving var, which keeps track of the serving - // status transmitted by the tablet. - servingStatus := make(chan bool, 1) - // timedout is accessed atomically because there could be a race - // between the goroutine that sets it and the check for its value - // later. - timedout := sync2.NewAtomicBool(false) - go func() { - for { - select { - case <-servingStatus: - continue - case <-time.After(hc.healthCheckTimeout): - timedout.Set(true) - streamCancel() - return - case <-streamCtx.Done(): - // If the stream is done, stop watching. - return - } - } - }() - - // Read stream health responses. - err := th.stream(streamCtx, func(shr *query.StreamHealthResponse) error { - // We received a message. Reset the back-off. - retryDelay = hc.retryDelay - // Don't block on send to avoid deadlocks. - select { - case servingStatus <- shr.Serving: - default: - } - return th.processResponse(hc, shr) - }) - - // streamCancel to make sure the watcher goroutine terminates. - streamCancel() - - if err != nil { - if strings.Contains(err.Error(), "health stats mismatch") { - hc.deleteTablet(th.Tablet) - return - } - res := th.SimpleCopy() - hc.broadcast(res) - } - // If there was a timeout send an error. We do this after stream has returned. - // This will ensure that this update prevails over any previous message that - // stream could have sent. - if timedout.Get() { - th.mu.Lock() - th.LastError = fmt.Errorf("healthcheck timed out (latest %v)", th.lastResponseTimestamp) - th.setServingState(false, th.LastError.Error()) - hcErrorCounters.Add([]string{th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)}, 1) - res := th.simpleCopyLocked() - th.mu.Unlock() - hc.broadcast(res) - } - - // Streaming RPC failed e.g. because vttablet was restarted or took too long. - // Sleep until the next retry is up or the context is done/canceled. - select { - case <-th.ctx.Done(): - return - case <-time.After(retryDelay): - // Exponentially back-off to prevent tight-loop. - retryDelay *= 2 - // Limit the retry delay backoff to the health check timeout - if retryDelay > hc.healthCheckTimeout { - retryDelay = hc.healthCheckTimeout - } - } - } -} - -func (th *tabletHealthCheck) closeConnection(ctx context.Context, err error) { - th.mu.Lock() - defer th.mu.Unlock() - log.Warningf("tablet %v healthcheck stream error: %v", th.Tablet.Alias, err) - th.setServingState(false, err.Error()) - th.LastError = err - _ = th.Conn.Close(ctx) - th.Conn = nil -} - -// finalizeConn closes the health checking connection. -// To be called only on exit from checkConn(). -func (th *tabletHealthCheck) finalizeConn() { - th.mu.Lock() - defer th.mu.Unlock() - th.setServingState(false, "finalizeConn closing connection") - // Note: checkConn() exits only when th.ctx.Done() is closed. Thus it's - // safe to simply get Err() value here and assign to LastError. - th.LastError = th.ctx.Err() - if th.Conn != nil { - // Don't use th.ctx because it's already closed. - // Use a separate context, and add a timeout to prevent unbounded waits. - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _ = th.Conn.Close(ctx) - th.Conn = nil - } -} diff --git a/go/vt/discovery/tablet_health_check.go b/go/vt/discovery/tablet_health_check.go new file mode 100644 index 00000000000..4d41d881a4f --- /dev/null +++ b/go/vt/discovery/tablet_health_check.go @@ -0,0 +1,324 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "vitess.io/vitess/go/sync2" + + "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/topotools" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vttablet/queryservice" + "vitess.io/vitess/go/vt/vttablet/tabletconn" + + "github.com/golang/protobuf/proto" + "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/proto/topodata" +) + +// tabletHealthCheck maintains the health status of a tablet. A map of this +// structure is maintained in HealthCheck. +type tabletHealthCheck struct { + ctx context.Context + // cancelFunc must be called before discarding tabletHealthCheck. + // This will ensure that the associated checkConn goroutine will terminate. + cancelFunc context.CancelFunc + // Tablet is the tablet object that was sent to HealthCheck.AddTablet. + Tablet *topodata.Tablet + mu sync.Mutex + // Conn is the connection associated with the tablet. + Conn queryservice.QueryService + // Target is the current target as returned by the streaming + // StreamHealth RPC. + Target *query.Target + // Serving describes if the tablet can be serving traffic. + Serving bool + // MasterTermStartTime is the last time at which + // this tablet was either elected the master, or received + // a TabletExternallyReparented event. It is set to 0 if the + // tablet doesn't think it's a master. + MasterTermStartTime int64 + // Stats is the current health status, as received by the + // StreamHealth RPC (replication lag, ...). + Stats *query.RealtimeStats + // LastError is the error we last saw when trying to get the + // tablet's healthcheck. + LastError error + // possibly delete both these + loggedServingState bool + lastResponseTimestamp time.Time // timestamp of the last healthcheck response +} + +// String is defined because we want to print a []*tabletHealthCheck array nicely. +func (th *tabletHealthCheck) String() string { + th.mu.Lock() + defer th.mu.Unlock() + return fmt.Sprintf("tabletHealthCheck{Tablet: %v,Target: %v,Serving: %v, MasterTermStartTime: %v, Stats: %v, LastError: %v", + th.Tablet, th.Target, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) +} + +// SimpleCopy returns a TabletHealth with all the necessary fields copied from tabletHealthCheck. +// Note that this is not a deep copy because we point to the same underlying RealtimeStats. +// That is fine because the RealtimeStats object is never changed after creation. +func (th *tabletHealthCheck) SimpleCopy() *TabletHealth { + th.mu.Lock() + defer th.mu.Unlock() + return th.simpleCopyLocked() +} + +func (th *tabletHealthCheck) simpleCopyLocked() *TabletHealth { + return &TabletHealth{ + Conn: th.Conn, + Tablet: th.Tablet, + Target: th.Target, + Stats: th.Stats, + LastError: th.LastError, + MasterTermStartTime: th.MasterTermStartTime, + Serving: th.Serving, + } +} + +// setServingState sets the tablet state to the given value. +// +// If the state changes, it logs the change so that failures +// from the health check connection are logged the first time, +// but don't continue to log if the connection stays down. +// +// th.mu must be locked before calling this function +func (th *tabletHealthCheck) setServingState(serving bool, reason string) { + if !th.loggedServingState || (serving != th.Serving) { + // Emit the log from a separate goroutine to avoid holding + // the th lock while logging is happening + go log.Infof("HealthCheckUpdate(Serving State): tablet: %v serving => %v for %v/%v (%v) reason: %s", + topotools.TabletIdent(th.Tablet), + serving, + th.Tablet.GetKeyspace(), + th.Tablet.GetShard(), + th.Target.GetTabletType(), + reason, + ) + th.loggedServingState = true + } + th.Serving = serving +} + +// stream streams healthcheck responses to callback. +func (th *tabletHealthCheck) stream(ctx context.Context, callback func(*query.StreamHealthResponse) error) error { + conn := th.getConnection() + if conn == nil { + // This signals the caller to retry + return nil + } + err := conn.StreamHealth(ctx, callback) + if err != nil { + // Depending on the specific error the caller can take action + th.closeConnection(ctx, err) + } + return err +} + +func (th *tabletHealthCheck) getConnection() queryservice.QueryService { + th.mu.Lock() + defer th.mu.Unlock() + if th.Conn == nil { + conn, err := tabletconn.GetDialer()(th.Tablet, grpcclient.FailFast(true)) + if err != nil { + th.LastError = err + return nil + } + th.Conn = conn + th.LastError = nil + } + return th.Conn +} + +// processResponse reads one health check response, and updates health +func (th *tabletHealthCheck) processResponse(hc *HealthCheck, shr *query.StreamHealthResponse) error { + select { + case <-th.ctx.Done(): + return th.ctx.Err() + default: + } + + // Check for invalid data, better than panicking. + if shr.Target == nil || shr.RealtimeStats == nil { + return fmt.Errorf("health stats is not valid: %v", shr) + } + + // an app-level error from tablet, force serving state. + var healthErr error + serving := shr.Serving + if shr.RealtimeStats.HealthError != "" { + healthErr = fmt.Errorf("vttablet error: %v", shr.RealtimeStats.HealthError) + serving = false + } + + if shr.TabletAlias != nil && !proto.Equal(shr.TabletAlias, th.Tablet.Alias) { + // TabletAlias change means that the host:port has been taken over by another tablet + // We could cancel / exit the healthcheck for this tablet right away + // However, we defer it until the next topo refresh informs us of the change because that is + // the only way to discover the new host/port + return vterrors.New(vtrpc.Code_FAILED_PRECONDITION, fmt.Sprintf("health stats mismatch, tablet %+v alias does not match response alias %v", th.Tablet, shr.TabletAlias)) + } + + th.mu.Lock() + currentTarget := th.Target + // check whether this is a trivial update so as to update healthy map + trivialNonMasterUpdate := th.LastError == nil && th.Serving && shr.RealtimeStats.HealthError == "" && shr.Serving && + currentTarget.TabletType != topodata.TabletType_MASTER && currentTarget.TabletType == shr.Target.TabletType + isMasterUpdate := shr.Target.TabletType == topodata.TabletType_MASTER + isMasterChange := th.Target.TabletType != topodata.TabletType_MASTER && shr.Target.TabletType == topodata.TabletType_MASTER + th.lastResponseTimestamp = time.Now() + th.Target = shr.Target + th.MasterTermStartTime = shr.TabletExternallyReparentedTimestamp + th.Stats = shr.RealtimeStats + th.LastError = healthErr + reason := "healthCheck update" + if healthErr != nil { + reason = "healthCheck update error: " + healthErr.Error() + } + th.setServingState(serving, reason) + th.mu.Unlock() + // notify downstream for master change + hc.updateHealth(th, shr, currentTarget, trivialNonMasterUpdate, isMasterUpdate, isMasterChange) + return nil +} + +// checkConn performs health checking on the given tablet. +func (th *tabletHealthCheck) checkConn(hc *HealthCheck) { + defer hc.connsWG.Done() + defer th.finalizeConn() + + retryDelay := hc.retryDelay + for { + streamCtx, streamCancel := context.WithCancel(th.ctx) + + // Setup a watcher that restarts the timer every time an update is received. + // If a timeout occurs for a serving tablet, we make it non-serving and send + // a status update. The stream is also terminated so it can be retried. + // servingStatus feeds into the serving var, which keeps track of the serving + // status transmitted by the tablet. + servingStatus := make(chan bool, 1) + // timedout is accessed atomically because there could be a race + // between the goroutine that sets it and the check for its value + // later. + timedout := sync2.NewAtomicBool(false) + go func() { + for { + select { + case <-servingStatus: + continue + case <-time.After(hc.healthCheckTimeout): + timedout.Set(true) + streamCancel() + return + case <-streamCtx.Done(): + // If the stream is done, stop watching. + return + } + } + }() + + // Read stream health responses. + err := th.stream(streamCtx, func(shr *query.StreamHealthResponse) error { + // We received a message. Reset the back-off. + retryDelay = hc.retryDelay + // Don't block on send to avoid deadlocks. + select { + case servingStatus <- shr.Serving: + default: + } + return th.processResponse(hc, shr) + }) + + // streamCancel to make sure the watcher goroutine terminates. + streamCancel() + + if err != nil { + if strings.Contains(err.Error(), "health stats mismatch") { + hc.deleteTablet(th.Tablet) + return + } + res := th.SimpleCopy() + hc.broadcast(res) + } + // If there was a timeout send an error. We do this after stream has returned. + // This will ensure that this update prevails over any previous message that + // stream could have sent. + if timedout.Get() { + th.mu.Lock() + th.LastError = fmt.Errorf("healthcheck timed out (latest %v)", th.lastResponseTimestamp) + th.setServingState(false, th.LastError.Error()) + hcErrorCounters.Add([]string{th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)}, 1) + res := th.simpleCopyLocked() + th.mu.Unlock() + hc.broadcast(res) + } + + // Streaming RPC failed e.g. because vttablet was restarted or took too long. + // Sleep until the next retry is up or the context is done/canceled. + select { + case <-th.ctx.Done(): + return + case <-time.After(retryDelay): + // Exponentially back-off to prevent tight-loop. + retryDelay *= 2 + // Limit the retry delay backoff to the health check timeout + if retryDelay > hc.healthCheckTimeout { + retryDelay = hc.healthCheckTimeout + } + } + } +} + +func (th *tabletHealthCheck) closeConnection(ctx context.Context, err error) { + th.mu.Lock() + defer th.mu.Unlock() + log.Warningf("tablet %v healthcheck stream error: %v", th.Tablet.Alias, err) + th.setServingState(false, err.Error()) + th.LastError = err + _ = th.Conn.Close(ctx) + th.Conn = nil +} + +// finalizeConn closes the health checking connection. +// To be called only on exit from checkConn(). +func (th *tabletHealthCheck) finalizeConn() { + th.mu.Lock() + defer th.mu.Unlock() + th.setServingState(false, "finalizeConn closing connection") + // Note: checkConn() exits only when th.ctx.Done() is closed. Thus it's + // safe to simply get Err() value here and assign to LastError. + th.LastError = th.ctx.Err() + if th.Conn != nil { + // Don't use th.ctx because it's already closed. + // Use a separate context, and add a timeout to prevent unbounded waits. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = th.Conn.Close(ctx) + th.Conn = nil + } +} From cb276c9f3f304abc1d56d393e0f543316820303d Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 25 May 2020 14:40:53 -0700 Subject: [PATCH 35/39] healthcheck: remove mu from tabletHealthCheck, replace more uses of tabletHealthCheck with TabletHealth, incorporate review comments to make code more readable, start using modern techniques in tests Signed-off-by: deepthi --- .../tabletgateway/healthcheck/main_test.go | 4 +- go/vt/discovery/healthcheck.go | 102 +++++---- go/vt/discovery/healthcheck_test.go | 108 +++++----- go/vt/discovery/legacy_healthcheck.go | 4 +- go/vt/discovery/replicationlag.go | 26 +-- go/vt/discovery/replicationlag_test.go | 61 +++--- go/vt/discovery/tablet_health_check.go | 201 +++++++++--------- go/vt/vtgate/executor.go | 8 +- 8 files changed, 254 insertions(+), 260 deletions(-) diff --git a/go/test/endtoend/tabletgateway/healthcheck/main_test.go b/go/test/endtoend/tabletgateway/healthcheck/main_test.go index 084586a9b3a..e2054fc2554 100644 --- a/go/test/endtoend/tabletgateway/healthcheck/main_test.go +++ b/go/test/endtoend/tabletgateway/healthcheck/main_test.go @@ -86,13 +86,13 @@ func TestMain(m *testing.M) { // Start vtgate vtgateInstance := clusterInstance.GetVtgateInstance() - // ensure it is torn down during cluster TearDown - clusterInstance.VtgateProcess = *vtgateInstance vtgateInstance.GatewayImplementation = "tabletgateway" err = vtgateInstance.Setup() if err != nil { return 1 } + // ensure it is torn down during cluster TearDown + clusterInstance.VtgateProcess = *vtgateInstance vtParams = mysql.ConnParams{ Host: clusterInstance.Hostname, Port: clusterInstance.VtgateMySQLPort, diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index bfbf86b67c8..3994b9b02f0 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -157,6 +157,9 @@ type TabletRecorder interface { ReplaceTablet(old, new *topodata.Tablet) } +type keyspaceShardTabletType string +type tabletAliasString string + // HealthCheck performs health checking and stores the results. // The goal of this object is to maintain a StreamHealth RPC // to a lot of tablets. Tablets are added / removed by calling the @@ -179,13 +182,13 @@ type HealthCheck struct { // mu protects all the following fields. mu sync.Mutex // authoritative map of tabletHealth by alias - healthByAlias map[string]*tabletHealthCheck + healthByAlias map[tabletAliasString]*tabletHealthCheck // a map keyed by keyspace.shard.tabletType - // contains a map of tabletHealthCheck keyed by tablet alias for each tablet relevant to the keyspace.shard.tabletType + // contains a map of TabletHealth keyed by tablet alias for each tablet relevant to the keyspace.shard.tabletType // has to be kept in sync with healthByAlias - healthData map[string]map[string]*tabletHealthCheck - // another map keyed by keyspace.shard.tabletType, this one containing a sorted list of tabletHealthCheck - healthy map[string][]*tabletHealthCheck + healthData map[keyspaceShardTabletType]map[tabletAliasString]*TabletHealth + // another map keyed by keyspace.shard.tabletType, this one containing a sorted list of TabletHealth + healthy map[keyspaceShardTabletType][]*TabletHealth // connsWG keeps track of all launched Go routines that monitor tablet connections. connsWG sync.WaitGroup // topology watchers that inform healthcheck of tablets being added and deleted @@ -221,9 +224,9 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur cell: localCell, retryDelay: retryDelay, healthCheckTimeout: healthCheckTimeout, - healthByAlias: make(map[string]*tabletHealthCheck), - healthData: make(map[string]map[string]*tabletHealthCheck), - healthy: make(map[string][]*tabletHealthCheck), + healthByAlias: make(map[tabletAliasString]*tabletHealthCheck), + healthData: make(map[keyspaceShardTabletType]map[tabletAliasString]*TabletHealth), + healthy: make(map[keyspaceShardTabletType][]*TabletHealth), subscribers: make(map[chan *TabletHealth]struct{}), cellAliases: make(map[string]string), } @@ -288,7 +291,7 @@ func (hc *HealthCheck) AddTablet(tablet *topodata.Tablet) { Shard: tablet.Shard, TabletType: tablet.Type, } - th := &tabletHealthCheck{ + thc := &tabletHealthCheck{ ctx: ctx, cancelFunc: cancelFunc, Tablet: tablet, @@ -298,24 +301,24 @@ func (hc *HealthCheck) AddTablet(tablet *topodata.Tablet) { // add to our datastore key := hc.keyFromTarget(target) tabletAlias := topoproto.TabletAliasString(tablet.Alias) - // TODO: can this ever already exist? - if _, ok := hc.healthByAlias[tabletAlias]; ok { - log.Errorf("Program bug") + if _, ok := hc.healthByAlias[tabletAliasString(tabletAlias)]; ok { + // We should not add a tablet that we already have + log.Errorf("Program bug: tried to add existing tablet: %v to healthcheck", tabletAlias) return } - hc.healthByAlias[tabletAlias] = th + hc.healthByAlias[tabletAliasString(tabletAlias)] = thc + res := thc.SimpleCopy() if ths, ok := hc.healthData[key]; !ok { - hc.healthData[key] = make(map[string]*tabletHealthCheck) - hc.healthData[key][tabletAlias] = th + hc.healthData[key] = make(map[tabletAliasString]*TabletHealth) + hc.healthData[key][tabletAliasString(tabletAlias)] = res } else { - // just overwrite it if it exists already? - ths[tabletAlias] = th + // just overwrite it if it exists already + ths[tabletAliasString(tabletAlias)] = res } - res := th.SimpleCopy() hc.broadcast(res) hc.connsWG.Add(1) - go th.checkConn(hc) + go thc.checkConn(hc) } // RemoveTablet removes the tablet, and stops the health check. @@ -338,7 +341,7 @@ func (hc *HealthCheck) deleteTablet(tablet *topodata.Tablet) { defer hc.mu.Unlock() key := hc.keyFromTablet(tablet) - tabletAlias := topoproto.TabletAliasString(tablet.Alias) + tabletAlias := tabletAliasString(topoproto.TabletAliasString(tablet.Alias)) // delete from authoritative map th, ok := hc.healthByAlias[tabletAlias] if !ok { @@ -358,31 +361,29 @@ func (hc *HealthCheck) deleteTablet(tablet *topodata.Tablet) { delete(ths, tabletAlias) } -func (hc *HealthCheck) updateHealth(th *tabletHealthCheck, shr *query.StreamHealthResponse, currentTarget *query.Target, trivialNonMasterUpdate bool, isMasterUpdate bool, isMasterChange bool) { +func (hc *HealthCheck) updateHealth(th *TabletHealth, shr *query.StreamHealthResponse, currentTarget *query.Target, trivialNonMasterUpdate bool, isMasterUpdate bool, isMasterChange bool) { // hc.healthByAlias is authoritative, it should be updated hc.mu.Lock() defer hc.mu.Unlock() - tabletAlias := topoproto.TabletAliasString(shr.TabletAlias) - // this will only change the first time, but it's easiest to set it always rather than check and set - hc.healthByAlias[tabletAlias] = th + tabletAlias := tabletAliasString(topoproto.TabletAliasString(shr.TabletAlias)) hcErrorCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard, topoproto.TabletTypeLString(shr.Target.TabletType)}, 0) + targetKey := hc.keyFromTarget(shr.Target) targetChanged := currentTarget.TabletType != shr.Target.TabletType || currentTarget.Keyspace != shr.Target.Keyspace || currentTarget.Shard != shr.Target.Shard if targetChanged { // keyspace and shard are not expected to change, but just in case ... // move this tabletHealthCheck to the correct map oldTargetKey := hc.keyFromTarget(currentTarget) - newTargetKey := hc.keyFromTarget(shr.Target) delete(hc.healthData[oldTargetKey], tabletAlias) - _, ok := hc.healthData[newTargetKey] + _, ok := hc.healthData[targetKey] if !ok { - hc.healthData[newTargetKey] = make(map[string]*tabletHealthCheck) + hc.healthData[targetKey] = make(map[tabletAliasString]*TabletHealth) } - hc.healthData[newTargetKey][tabletAlias] = th } + // add it to the map by target + hc.healthData[targetKey][tabletAlias] = th - targetKey := hc.keyFromTarget(shr.Target) if isMasterUpdate { if len(hc.healthy[targetKey]) == 0 { hc.healthy[targetKey] = append(hc.healthy[targetKey], th) @@ -405,7 +406,7 @@ func (hc *HealthCheck) updateHealth(th *tabletHealthCheck, shr *query.StreamHeal if !trivialNonMasterUpdate { if shr.Target.TabletType != topodata.TabletType_MASTER { all := hc.healthData[targetKey] - allArray := make([]*tabletHealthCheck, 0, len(all)) + allArray := make([]*TabletHealth, 0, len(all)) for _, s := range all { allArray = append(allArray, s) } @@ -414,20 +415,19 @@ func (hc *HealthCheck) updateHealth(th *tabletHealthCheck, shr *query.StreamHeal if targetChanged && currentTarget.TabletType != topodata.TabletType_MASTER { // also recompute old target's healthy list oldTargetKey := hc.keyFromTarget(currentTarget) all := hc.healthData[oldTargetKey] - allArray := make([]*tabletHealthCheck, 0, len(all)) + allArray := make([]*TabletHealth, 0, len(all)) for _, s := range all { allArray = append(allArray, s) } hc.healthy[oldTargetKey] = FilterStatsByReplicationLag(allArray) } } - result := th.SimpleCopy() if isMasterChange { log.Errorf("Adding 1 to MasterPromoted counter for tablet: %v, shr.Tablet: %v, shr.TabletType: %v", currentTarget, topoproto.TabletAliasString(shr.TabletAlias), shr.Target.TabletType) hcMasterPromotedCounters.Add([]string{shr.Target.Keyspace, shr.Target.Shard}, 1) } // broadcast to subscribers - hc.broadcast(result) + hc.broadcast(th) } @@ -525,10 +525,7 @@ func (hc *HealthCheck) GetHealthyTabletStats(target *query.Target) []*TabletHeal var result []*TabletHealth hc.mu.Lock() defer hc.mu.Unlock() - for _, thc := range hc.healthy[hc.keyFromTarget(target)] { - result = append(result, thc.SimpleCopy()) - } - return result + return append(result, hc.healthy[hc.keyFromTarget(target)]...) } // getTabletStats returns all tablets for the given target. @@ -541,7 +538,7 @@ func (hc *HealthCheck) getTabletStats(target *query.Target) []*TabletHealth { defer hc.mu.Unlock() ths := hc.healthData[hc.keyFromTarget(target)] for _, th := range ths { - result = append(result, th.SimpleCopy()) + result = append(result, th) } return result } @@ -578,13 +575,13 @@ func (hc *HealthCheck) waitForTablets(ctx context.Context, targets []*query.Targ continue } - var stats []*TabletHealth + var tabletHealths []*TabletHealth if requireServing { - stats = hc.GetHealthyTabletStats(target) + tabletHealths = hc.GetHealthyTabletStats(target) } else { - stats = hc.getTabletStats(target) + tabletHealths = hc.getTabletStats(target) } - if len(stats) == 0 { + if len(tabletHealths) == 0 { allPresent = false } else { targets[i] = nil @@ -609,12 +606,12 @@ func (hc *HealthCheck) waitForTablets(ctx context.Context, targets []*query.Targ // Target includes cell which we ignore here // because tabletStatsCache is intended to be per-cell -func (hc *HealthCheck) keyFromTarget(target *query.Target) string { - return fmt.Sprintf("%s.%s.%d", target.Keyspace, target.Shard, target.TabletType) +func (hc *HealthCheck) keyFromTarget(target *query.Target) keyspaceShardTabletType { + return keyspaceShardTabletType(fmt.Sprintf("%s.%s.%s", target.Keyspace, target.Shard, topoproto.TabletTypeLString(target.TabletType))) } -func (hc *HealthCheck) keyFromTablet(tablet *topodata.Tablet) string { - return fmt.Sprintf("%s.%s.%d", tablet.Keyspace, tablet.Shard, tablet.Type) +func (hc *HealthCheck) keyFromTablet(tablet *topodata.Tablet) keyspaceShardTabletType { + return keyspaceShardTabletType(fmt.Sprintf("%s.%s.%s", tablet.Keyspace, tablet.Shard, topoproto.TabletTypeLString(tablet.Type))) } func (hc *HealthCheck) getAliasByCell(cell string) string { @@ -714,15 +711,12 @@ func (hc *HealthCheck) servingConnStats() map[string]int64 { res := make(map[string]int64) hc.mu.Lock() defer hc.mu.Unlock() - for _, th := range hc.healthByAlias { - th.mu.Lock() - if !th.Serving || th.LastError != nil { - th.mu.Unlock() - continue + for key, ths := range hc.healthData { + for _, th := range ths { + if th.Serving && th.LastError == nil { + res[string(key)]++ + } } - key := fmt.Sprintf("%s.%s.%s", th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)) - th.mu.Unlock() - res[key]++ } return res } diff --git a/go/vt/discovery/healthcheck_test.go b/go/vt/discovery/healthcheck_test.go index 080b3f07025..2e638a53acd 100644 --- a/go/vt/discovery/healthcheck_test.go +++ b/go/vt/discovery/healthcheck_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/vttablet/queryservice/fakes" "github.com/stretchr/testify/assert" @@ -78,7 +79,7 @@ func TestHealthCheck(t *testing.T) { MasterTermStartTime: 0, } result := <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") shr := &querypb.StreamHealthResponse{ TabletAlias: tablet.Alias, @@ -97,7 +98,7 @@ func TestHealthCheck(t *testing.T) { MasterTermStartTime: 0, } // create a context with timeout and select on it and channel - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") tcsl := hc.CacheStatus() tcslWant := TabletsCacheStatusList{{ @@ -138,7 +139,7 @@ func TestHealthCheck(t *testing.T) { input <- shr result = <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") testChecksum(t, 1780128002, hc.stateChecksum()) err := checkErrorCounter("k", "s", topodatapb.TabletType_MASTER, 0) @@ -161,7 +162,7 @@ func TestHealthCheck(t *testing.T) { } input <- shr result = <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") testChecksum(t, 1027934207, hc.stateChecksum()) // HealthError @@ -182,6 +183,7 @@ func TestHealthCheck(t *testing.T) { } input <- shr result = <-resultChan + //TODO: figure out how to compare objects that contain errors using utils.MustMatch assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) testChecksum(t, 1027934207, hc.stateChecksum()) // unchanged @@ -211,7 +213,7 @@ func TestHealthCheckStreamError(t *testing.T) { MasterTermStartTime: 0, } result := <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") // one tablet after receiving a StreamHealthResponse shr := &querypb.StreamHealthResponse{ @@ -230,7 +232,7 @@ func TestHealthCheckStreamError(t *testing.T) { } input <- shr result = <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") // Stream error fc.errCh <- fmt.Errorf("some stream error") @@ -243,6 +245,7 @@ func TestHealthCheckStreamError(t *testing.T) { LastError: fmt.Errorf("some stream error"), } result = <-resultChan + //TODO: figure out how to compare objects that contain errors using utils.MustMatch assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) } @@ -267,7 +270,7 @@ func TestHealthCheckVerifiesTabletAlias(t *testing.T) { MasterTermStartTime: 0, } result := <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") input <- &querypb.StreamHealthResponse{ Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, @@ -309,7 +312,7 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { MasterTermStartTime: 0, } result := <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") // one tablet after receiving a StreamHealthResponse shr := &querypb.StreamHealthResponse{ @@ -328,7 +331,7 @@ func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) { } input <- shr result = <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") // Change input to distinguish between stats sent before and after Close(). shr.TabletExternallyReparentedTimestamp = 11 @@ -370,7 +373,7 @@ func TestHealthCheckTimeout(t *testing.T) { MasterTermStartTime: 0, } result := <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") // one tablet after receiving a StreamHealthResponse shr := &querypb.StreamHealthResponse{ @@ -389,7 +392,7 @@ func TestHealthCheckTimeout(t *testing.T) { } input <- shr result = <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") assert.Nil(t, checkErrorCounter("k", "s", topodatapb.TabletType_REPLICA, 0)) // wait for timeout period @@ -415,7 +418,7 @@ func TestHealthCheckTimeout(t *testing.T) { // wait for the exponential backoff to wear off and health monitoring to resume. result = <-resultChan - assert.True(t, want.DeepEqual(result), "Wrong TabletHealth data\n Expected: %v\n Actual: %v", want, result) + mustMatch(t, want, result, "Wrong TabletHealth data") } // TestGetHealthyTablets tests the functionality of GetHealthyTabletStats. @@ -438,7 +441,7 @@ func TestGetHealthyTablets(t *testing.T) { <-resultChan // empty a := hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) - assert.Equal(t, 0, len(a), "wrong result, expected empty list") + assert.Empty(t, a, "wrong result, expected empty list") shr := &querypb.StreamHealthResponse{ TabletAlias: tablet.Alias, @@ -447,19 +450,18 @@ func TestGetHealthyTablets(t *testing.T) { TabletExternallyReparentedTimestamp: 0, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, } - want := &TabletHealth{ + want := []*TabletHealth{{ Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, MasterTermStartTime: 0, - } + }} input <- shr <-resultChan // check it's there a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) - assert.Equal(t, 1, len(a), "Wrong number of results") - assert.True(t, want.DeepEqual(a[0]), "unexpected result") + mustMatch(t, want, a, "unexpected result") // update health with a change that won't change health array shr = &querypb.StreamHealthResponse{ @@ -473,16 +475,8 @@ func TestGetHealthyTablets(t *testing.T) { // wait for result before checking <-resultChan // check it's there - want = &TabletHealth{ - Tablet: tablet, - Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, - Serving: true, - Stats: &querypb.RealtimeStats{SecondsBehindMaster: 2, CpuUsage: 0.2}, - MasterTermStartTime: 0, - } a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) - assert.Equal(t, 1, len(a), "Wrong number of results") - assert.True(t, want.DeepEqual(a[0]), "unexpected result") + mustMatch(t, want, a, "unexpected result") // update stats with a change that will change health array shr = &querypb.StreamHealthResponse{ @@ -492,20 +486,19 @@ func TestGetHealthyTablets(t *testing.T) { TabletExternallyReparentedTimestamp: 0, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 35, CpuUsage: 0.2}, } - want = &TabletHealth{ + want = []*TabletHealth{{ Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 35, CpuUsage: 0.2}, MasterTermStartTime: 0, - } + }} input <- shr // wait for result before checking <-resultChan // check it's there a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) - assert.Equal(t, 1, len(a), "Wrong number of results") - assert.True(t, want.DeepEqual(a[0]), "unexpected result") + mustMatch(t, want, a, "unexpected result") // add a second tablet tablet2 := topo.NewTablet(11, "cell", "host2") @@ -528,13 +521,19 @@ func TestGetHealthyTablets(t *testing.T) { TabletExternallyReparentedTimestamp: 0, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, } - want2 := &TabletHealth{ + want2 := []*TabletHealth{{ + Tablet: tablet, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 35, CpuUsage: 0.2}, + MasterTermStartTime: 0, + }, { Tablet: tablet2, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, MasterTermStartTime: 0, - } + }} input2 <- shr2 // wait for result <-resultChan @@ -543,8 +542,7 @@ func TestGetHealthyTablets(t *testing.T) { if a[0].Tablet.Alias.Uid == 11 { a[0], a[1] = a[1], a[0] } - assert.True(t, want.DeepEqual(a[0]), "unexpected result") - assert.True(t, want2.DeepEqual(a[1]), "unexpected result") + mustMatch(t, want2, a, "unexpected result") shr2 = &querypb.StreamHealthResponse{ TabletAlias: tablet2.Alias, @@ -558,7 +556,6 @@ func TestGetHealthyTablets(t *testing.T) { <-resultChan a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) assert.Equal(t, 1, len(a), "Wrong number of results") - assert.True(t, want.DeepEqual(a[0]), "unexpected result") // second tablet turns into a master shr2 = &querypb.StreamHealthResponse{ @@ -573,20 +570,18 @@ func TestGetHealthyTablets(t *testing.T) { <-resultChan // check we only have 1 healthy replica left a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) - assert.Equal(t, 1, len(a), "Wrong number of results") - assert.True(t, want.DeepEqual(a[0]), "unexpected result") + mustMatch(t, want, a, "unexpected result") - want2 = &TabletHealth{ + want2 = []*TabletHealth{{ Tablet: tablet2, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 0, CpuUsage: 0.2}, MasterTermStartTime: 10, - } + }} // check we have a master now a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) - assert.Equal(t, 1, len(a), "Wrong number of results") - assert.True(t, want2.DeepEqual(a[0]), "unexpected result") + mustMatch(t, want2, a, "unexpected result") // reparent: old replica goes into master shr = &querypb.StreamHealthResponse{ @@ -598,27 +593,25 @@ func TestGetHealthyTablets(t *testing.T) { } input <- shr <-resultChan - want = &TabletHealth{ + want = []*TabletHealth{{ Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}, Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 0, CpuUsage: 0.2}, MasterTermStartTime: 20, - } + }} // check we lost all replicas, and master is new one a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) - assert.Equal(t, 0, len(a), "Wrong number of results") + assert.Empty(t, a, "Wrong number of results") a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) - assert.Equal(t, 1, len(a), "Wrong number of results") - assert.True(t, want.DeepEqual(a[0]), "unexpected result") + mustMatch(t, want, a, "unexpected result") // old master sending an old ping should be ignored input2 <- shr2 <-resultChan a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER}) - assert.Equal(t, 1, len(a), "Wrong number of results") - assert.True(t, want.DeepEqual(a[0]), "unexpected result") + mustMatch(t, want, a, "unexpected result") } func TestAliases(t *testing.T) { @@ -666,13 +659,13 @@ func TestAliases(t *testing.T) { TabletExternallyReparentedTimestamp: 0, RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, } - want := &TabletHealth{ + want := []*TabletHealth{{ Tablet: tablet, Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, MasterTermStartTime: 0, - } + }} input <- shr ticker = time.NewTicker(1 * time.Second) @@ -686,8 +679,7 @@ func TestAliases(t *testing.T) { // check it's there a := hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) - assert.Equal(t, 1, len(a), "Wrong number of results") - assert.True(t, want.DeepEqual(a[0]), "unexpected result") + mustMatch(t, want, a, "Wrong TabletHealth data") // add another tablet in a diff cell, diff region tablet2 := topo.NewTablet(2, "cell2", "host4") @@ -710,8 +702,7 @@ func TestAliases(t *testing.T) { // check that we still have only tablet in healthy list a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}) - assert.Equal(t, 1, len(a), "Wrong number of results") - assert.True(t, want.DeepEqual(a[0]), "unexpected result") + mustMatch(t, want, a, "Wrong TabletHealth data") } func TestTemplate(t *testing.T) { @@ -767,7 +758,7 @@ func TestDebugURLFormatting(t *testing.T) { require.Contains(t, wr.String(), expectedURL, "output missing formatted URL") } -func tabletDialer(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) { +func tabletDialer(tablet *topodatapb.Tablet, _ grpcclient.FailFast) (queryservice.QueryService, error) { key := TabletToMapKey(tablet) if qs, ok := connMap[key]; ok { return qs, nil @@ -860,3 +851,10 @@ func checkErrorCounter(keyspace, shard string, tabletType topodatapb.TabletType, } return nil } + +var mustMatch = utils.MustMatchFn( + []interface{}{ // types with unexported fields + TabletHealth{}, + }, + []string{".Conn"}, // ignored fields +) diff --git a/go/vt/discovery/legacy_healthcheck.go b/go/vt/discovery/legacy_healthcheck.go index c6edb06df55..69522fd8555 100644 --- a/go/vt/discovery/legacy_healthcheck.go +++ b/go/vt/discovery/legacy_healthcheck.go @@ -928,8 +928,8 @@ func (hc *LegacyHealthCheckImpl) cacheStatusMap() map[string]*LegacyTabletsCache } tcsMap[key] = tcs } - stats := th.latestTabletStats - tcs.TabletsStats = append(tcs.TabletsStats, &stats) + tabletStats := th.latestTabletStats + tcs.TabletsStats = append(tcs.TabletsStats, &tabletStats) } return tcsMap } diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index 28f55fa6f90..b448d8ece7d 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -32,18 +32,18 @@ var ( // IsReplicationLagHigh verifies that the given LegacytabletHealth refers to a tablet with high // replication lag, i.e. higher than the configured discovery_low_replication_lag flag. -func IsReplicationLagHigh(tabletHealth *tabletHealthCheck) bool { +func IsReplicationLagHigh(tabletHealth *TabletHealth) bool { return float64(tabletHealth.Stats.SecondsBehindMaster) > lowReplicationLag.Seconds() } // IsReplicationLagVeryHigh verifies that the given LegacytabletHealth refers to a tablet with very high // replication lag, i.e. higher than the configured discovery_high_replication_lag_minimum_serving flag. -func IsReplicationLagVeryHigh(tabletHealth *tabletHealthCheck) bool { +func IsReplicationLagVeryHigh(tabletHealth *TabletHealth) bool { return float64(tabletHealth.Stats.SecondsBehindMaster) > highReplicationLagMinServing.Seconds() } -// FilterStatsByReplicationLag filters the list of tabletHealthCheck by tabletHealthCheck.Stats.SecondsBehindMaster. -// Note that tabletHealthCheck that is non-serving or has error is ignored. +// FilterStatsByReplicationLag filters the list of TabletHealth by TabletHealth.Stats.SecondsBehindMaster. +// Note that TabletHealth that is non-serving or has error is ignored. // // The simplified logic: // - Return tablets that have lag <= lowReplicationLag. @@ -67,7 +67,7 @@ func IsReplicationLagVeryHigh(tabletHealth *tabletHealthCheck) bool { // The default for this is 2h, same as the discovery_high_replication_lag_minimum_serving here. // * degraded_threshold: this is only used by vttablet for display. It should match // discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. -func FilterStatsByReplicationLag(tabletHealthList []*tabletHealthCheck) []*tabletHealthCheck { +func FilterStatsByReplicationLag(tabletHealthList []*TabletHealth) []*TabletHealth { if !*legacyReplicationLagAlgorithm { return filterStatsByLag(tabletHealthList) } @@ -81,7 +81,7 @@ func FilterStatsByReplicationLag(tabletHealthList []*tabletHealthCheck) []*table } -func filterStatsByLag(tabletHealthList []*tabletHealthCheck) []*tabletHealthCheck { +func filterStatsByLag(tabletHealthList []*TabletHealth) []*TabletHealth { list := make([]tabletLagSnapshot, 0, len(tabletHealthList)) // filter non-serving tablets and those with very high replication lag for _, ts := range tabletHealthList { @@ -98,7 +98,7 @@ func filterStatsByLag(tabletHealthList []*tabletHealthCheck) []*tabletHealthChec sort.Sort(tabletLagSnapshotList(list)) // Pick those with low replication lag, but at least minNumTablets tablets regardless. - res := make([]*tabletHealthCheck, 0, len(list)) + res := make([]*TabletHealth, 0, len(list)) for i := 0; i < len(list); i++ { if !IsReplicationLagHigh(list[i].ts) || i < *minNumTablets { res = append(res, list[i].ts) @@ -107,8 +107,8 @@ func filterStatsByLag(tabletHealthList []*tabletHealthCheck) []*tabletHealthChec return res } -func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*tabletHealthCheck) []*tabletHealthCheck { - list := make([]*tabletHealthCheck, 0, len(tabletHealthList)) +func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*TabletHealth) []*TabletHealth { + list := make([]*TabletHealth, 0, len(tabletHealthList)) // filter non-serving tablets for _, ts := range tabletHealthList { if !ts.Serving || ts.LastError != nil || ts.Stats == nil { @@ -132,7 +132,7 @@ func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*tabletHealthCheck) } // filter those affecting "mean" lag significantly // calculate mean for all tablets - res := make([]*tabletHealthCheck, 0, len(list)) + res := make([]*TabletHealth, 0, len(list)) m, _ := mean(list, -1) for i, ts := range list { // calculate mean by excluding ith tablet @@ -174,7 +174,7 @@ func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*tabletHealthCheck) sort.Sort(byReplag(snapshots)) // Pick the first minNumTablets tablets. - res = make([]*tabletHealthCheck, 0, *minNumTablets) + res = make([]*TabletHealth, 0, *minNumTablets) for i := 0; i < min(*minNumTablets, len(snapshots)); i++ { res = append(res, snapshots[i].ts) } @@ -188,7 +188,7 @@ func (a byReplag) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byReplag) Less(i, j int) bool { return a[i].replag < a[j].replag } type tabletLagSnapshot struct { - ts *tabletHealthCheck + ts *TabletHealth replag uint32 } type tabletLagSnapshotList []tabletLagSnapshot @@ -206,7 +206,7 @@ func min(a, b int) int { // mean calculates the mean value over the given list, // while excluding the item with the specified index. -func mean(tabletHealthList []*tabletHealthCheck, idxExclude int) (uint64, error) { +func mean(tabletHealthList []*TabletHealth, idxExclude int) (uint64, error) { var sum uint64 var count uint64 for i, ts := range tabletHealthList { diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index c95eeffa1be..2c6bc45cc0f 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -33,18 +33,18 @@ func testSetMinNumTablets(newMin int) { func TestFilterByReplicationLagUnhealthy(t *testing.T) { // 1 healthy serving tablet, 1 not healhty - ts1 := &tabletHealthCheck{ + ts1 := &TabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{}, } - ts2 := &tabletHealthCheck{ + ts2 := &TabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: false, Stats: &querypb.RealtimeStats{}, } - got := FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2}) - want := []*tabletHealthCheck{ts1} + got := FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2}) + want := []*TabletHealth{ts1} mustMatch(t, want, got, "FilterStatsByReplicationLag") } @@ -100,9 +100,9 @@ func TestFilterByReplicationLag(t *testing.T) { } for _, tc := range cases { - lts := make([]*tabletHealthCheck, len(tc.input)) + lts := make([]*TabletHealth, len(tc.input)) for i, lag := range tc.input { - lts[i] = &tabletHealthCheck{ + lts[i] = &TabletHealth{ Tablet: topo.NewTablet(uint32(i+1), "cell", fmt.Sprintf("host-%vs-behind", lag)), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: lag}, @@ -128,53 +128,53 @@ func TestFilterByReplicationLagThreeTabletMin(t *testing.T) { // Use at least 3 tablets if possible testSetMinNumTablets(3) // lags of (1s, 1s, 10m, 11m) - returns at least32 items where the slightly delayed ones that are returned are the 10m and 11m ones. - ts1 := &tabletHealthCheck{ + ts1 := &TabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &tabletHealthCheck{ + ts2 := &TabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts3 := &tabletHealthCheck{ + ts3 := &TabletHealth{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts4 := &tabletHealthCheck{ + ts4 := &TabletHealth{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - got := FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2, ts3, ts4}) - want := []*tabletHealthCheck{ts1, ts2, ts3} + got := FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2, ts3, ts4}) + want := []*TabletHealth{ts1, ts2, ts3} mustMatch(t, want, got, "FilterStatsByReplicationLag") // lags of (11m, 10m, 1s, 1s) - reordered tablets returns the same 3 items where the slightly delayed one that is returned is the 10m and 11m ones. - ts1 = &tabletHealthCheck{ + ts1 = &TabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 11 * 60}, } - ts2 = &tabletHealthCheck{ + ts2 = &TabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60}, } - ts3 = &tabletHealthCheck{ + ts3 = &TabletHealth{ Tablet: topo.NewTablet(3, "cell", "host3"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts4 = &tabletHealthCheck{ + ts4 = &TabletHealth{ Tablet: topo.NewTablet(4, "cell", "host4"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - got = FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2, ts3, ts4}) - want = []*tabletHealthCheck{ts3, ts4, ts2} + got = FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2, ts3, ts4}) + want = []*TabletHealth{ts3, ts4, ts2} mustMatch(t, want, got, "FilterStatsByReplicationLag") // Reset to the default testSetMinNumTablets(2) @@ -184,41 +184,34 @@ func TestFilterStatsByReplicationLagOneTabletMin(t *testing.T) { // Use at least 1 tablets if possible testSetMinNumTablets(1) // lags of (1s, 100m) - return only healthy tablet if that is all that is available. - ts1 := &tabletHealthCheck{ + ts1 := &TabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1}, } - ts2 := &tabletHealthCheck{ + ts2 := &TabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got := FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2}) - want := []*tabletHealthCheck{ts1} + got := FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2}) + want := []*TabletHealth{ts1} mustMatch(t, want, got, "FilterStatsByReplicationLag") // lags of (1m, 100m) - return only healthy tablet if that is all that is healthy enough. - ts1 = &tabletHealthCheck{ + ts1 = &TabletHealth{ Tablet: topo.NewTablet(1, "cell", "host1"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60}, } - ts2 = &tabletHealthCheck{ + ts2 = &TabletHealth{ Tablet: topo.NewTablet(2, "cell", "host2"), Serving: true, Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60}, } - got = FilterStatsByReplicationLag([]*tabletHealthCheck{ts1, ts2}) - want = []*tabletHealthCheck{ts1} - mustMatch(t, want, got, "FilterStatsByReplicationLag") + got = FilterStatsByReplicationLag([]*TabletHealth{ts1, ts2}) + want = []*TabletHealth{ts1} + utils.MustMatch(t, want, got, "FilterStatsByReplicationLag") // Reset to the default testSetMinNumTablets(2) } - -var mustMatch = utils.MustMatchFn( - []interface{}{ // types with unexported fields - tabletHealthCheck{}, - }, - []string{".mu"}, // ignored fields -) diff --git a/go/vt/discovery/tablet_health_check.go b/go/vt/discovery/tablet_health_check.go index 4d41d881a4f..39aa9895c0c 100644 --- a/go/vt/discovery/tablet_health_check.go +++ b/go/vt/discovery/tablet_health_check.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "strings" - "sync" "time" "vitess.io/vitess/go/sync2" @@ -48,7 +47,6 @@ type tabletHealthCheck struct { cancelFunc context.CancelFunc // Tablet is the tablet object that was sent to HealthCheck.AddTablet. Tablet *topodata.Tablet - mu sync.Mutex // Conn is the connection associated with the tablet. Conn queryservice.QueryService // Target is the current target as returned by the streaming @@ -73,31 +71,23 @@ type tabletHealthCheck struct { } // String is defined because we want to print a []*tabletHealthCheck array nicely. -func (th *tabletHealthCheck) String() string { - th.mu.Lock() - defer th.mu.Unlock() +func (thc *tabletHealthCheck) String() string { return fmt.Sprintf("tabletHealthCheck{Tablet: %v,Target: %v,Serving: %v, MasterTermStartTime: %v, Stats: %v, LastError: %v", - th.Tablet, th.Target, th.Serving, th.MasterTermStartTime, *th.Stats, th.LastError) + thc.Tablet, thc.Target, thc.Serving, thc.MasterTermStartTime, *thc.Stats, thc.LastError) } // SimpleCopy returns a TabletHealth with all the necessary fields copied from tabletHealthCheck. // Note that this is not a deep copy because we point to the same underlying RealtimeStats. // That is fine because the RealtimeStats object is never changed after creation. -func (th *tabletHealthCheck) SimpleCopy() *TabletHealth { - th.mu.Lock() - defer th.mu.Unlock() - return th.simpleCopyLocked() -} - -func (th *tabletHealthCheck) simpleCopyLocked() *TabletHealth { +func (thc *tabletHealthCheck) SimpleCopy() *TabletHealth { return &TabletHealth{ - Conn: th.Conn, - Tablet: th.Tablet, - Target: th.Target, - Stats: th.Stats, - LastError: th.LastError, - MasterTermStartTime: th.MasterTermStartTime, - Serving: th.Serving, + Conn: thc.Conn, + Tablet: thc.Tablet, + Target: thc.Target, + Stats: thc.Stats, + LastError: thc.LastError, + MasterTermStartTime: thc.MasterTermStartTime, + Serving: thc.Serving, } } @@ -107,27 +97,27 @@ func (th *tabletHealthCheck) simpleCopyLocked() *TabletHealth { // from the health check connection are logged the first time, // but don't continue to log if the connection stays down. // -// th.mu must be locked before calling this function -func (th *tabletHealthCheck) setServingState(serving bool, reason string) { - if !th.loggedServingState || (serving != th.Serving) { +// thc.mu must be locked before calling this function +func (thc *tabletHealthCheck) setServingState(serving bool, reason string) { + if !thc.loggedServingState || (serving != thc.Serving) { // Emit the log from a separate goroutine to avoid holding // the th lock while logging is happening go log.Infof("HealthCheckUpdate(Serving State): tablet: %v serving => %v for %v/%v (%v) reason: %s", - topotools.TabletIdent(th.Tablet), + topotools.TabletIdent(thc.Tablet), serving, - th.Tablet.GetKeyspace(), - th.Tablet.GetShard(), - th.Target.GetTabletType(), + thc.Tablet.GetKeyspace(), + thc.Tablet.GetShard(), + thc.Target.GetTabletType(), reason, ) - th.loggedServingState = true + thc.loggedServingState = true } - th.Serving = serving + thc.Serving = serving } // stream streams healthcheck responses to callback. -func (th *tabletHealthCheck) stream(ctx context.Context, callback func(*query.StreamHealthResponse) error) error { - conn := th.getConnection() +func (thc *tabletHealthCheck) stream(ctx context.Context, callback func(*query.StreamHealthResponse) error) error { + conn := thc.getConnection() if conn == nil { // This signals the caller to retry return nil @@ -135,31 +125,29 @@ func (th *tabletHealthCheck) stream(ctx context.Context, callback func(*query.St err := conn.StreamHealth(ctx, callback) if err != nil { // Depending on the specific error the caller can take action - th.closeConnection(ctx, err) + thc.closeConnection(ctx, err) } return err } -func (th *tabletHealthCheck) getConnection() queryservice.QueryService { - th.mu.Lock() - defer th.mu.Unlock() - if th.Conn == nil { - conn, err := tabletconn.GetDialer()(th.Tablet, grpcclient.FailFast(true)) +func (thc *tabletHealthCheck) getConnection() queryservice.QueryService { + if thc.Conn == nil { + conn, err := tabletconn.GetDialer()(thc.Tablet, grpcclient.FailFast(true)) if err != nil { - th.LastError = err + thc.LastError = err return nil } - th.Conn = conn - th.LastError = nil + thc.Conn = conn + thc.LastError = nil } - return th.Conn + return thc.Conn } // processResponse reads one health check response, and updates health -func (th *tabletHealthCheck) processResponse(hc *HealthCheck, shr *query.StreamHealthResponse) error { +func (thc *tabletHealthCheck) processResponse(hc *HealthCheck, shr *query.StreamHealthResponse) error { select { - case <-th.ctx.Done(): - return th.ctx.Err() + case <-thc.ctx.Done(): + return thc.ctx.Err() default: } @@ -176,45 +164,73 @@ func (th *tabletHealthCheck) processResponse(hc *HealthCheck, shr *query.StreamH serving = false } - if shr.TabletAlias != nil && !proto.Equal(shr.TabletAlias, th.Tablet.Alias) { + if shr.TabletAlias != nil && !proto.Equal(shr.TabletAlias, thc.Tablet.Alias) { // TabletAlias change means that the host:port has been taken over by another tablet - // We could cancel / exit the healthcheck for this tablet right away - // However, we defer it until the next topo refresh informs us of the change because that is - // the only way to discover the new host/port - return vterrors.New(vtrpc.Code_FAILED_PRECONDITION, fmt.Sprintf("health stats mismatch, tablet %+v alias does not match response alias %v", th.Tablet, shr.TabletAlias)) + // We cancel / exit the healthcheck for this tablet right away + // With the next topo refresh we will get a new tablet with the new host/port + return vterrors.New(vtrpc.Code_FAILED_PRECONDITION, fmt.Sprintf("health stats mismatch, tablet %+v alias does not match response alias %v", thc.Tablet, shr.TabletAlias)) } - th.mu.Lock() - currentTarget := th.Target + currentTarget := thc.Target // check whether this is a trivial update so as to update healthy map - trivialNonMasterUpdate := th.LastError == nil && th.Serving && shr.RealtimeStats.HealthError == "" && shr.Serving && - currentTarget.TabletType != topodata.TabletType_MASTER && currentTarget.TabletType == shr.Target.TabletType + trivialNonMasterUpdate := thc.LastError == nil && thc.Serving && shr.RealtimeStats.HealthError == "" && shr.Serving && + currentTarget.TabletType != topodata.TabletType_MASTER && currentTarget.TabletType == shr.Target.TabletType && thc.isTrivialReplagChange(shr.RealtimeStats) isMasterUpdate := shr.Target.TabletType == topodata.TabletType_MASTER - isMasterChange := th.Target.TabletType != topodata.TabletType_MASTER && shr.Target.TabletType == topodata.TabletType_MASTER - th.lastResponseTimestamp = time.Now() - th.Target = shr.Target - th.MasterTermStartTime = shr.TabletExternallyReparentedTimestamp - th.Stats = shr.RealtimeStats - th.LastError = healthErr + isMasterChange := thc.Target.TabletType != topodata.TabletType_MASTER && shr.Target.TabletType == topodata.TabletType_MASTER + thc.lastResponseTimestamp = time.Now() + thc.Target = shr.Target + thc.MasterTermStartTime = shr.TabletExternallyReparentedTimestamp + thc.Stats = shr.RealtimeStats + thc.LastError = healthErr reason := "healthCheck update" if healthErr != nil { reason = "healthCheck update error: " + healthErr.Error() } - th.setServingState(serving, reason) - th.mu.Unlock() + thc.setServingState(serving, reason) + // notify downstream for master change - hc.updateHealth(th, shr, currentTarget, trivialNonMasterUpdate, isMasterUpdate, isMasterChange) + hc.updateHealth(thc.SimpleCopy(), shr, currentTarget, trivialNonMasterUpdate, isMasterUpdate, isMasterChange) return nil } +// isTrivialReplagChange returns true iff the old and new RealtimeStats +// haven't changed enough to warrant re-calling FilterLegacyStatsByReplicationLag. +func (thc *tabletHealthCheck) isTrivialReplagChange(newStats *query.RealtimeStats) bool { + // first time always return false + if thc.Stats == nil { + return false + } + // Skip replag filter when replag remains in the low rep lag range, + // which should be the case majority of the time. + lowRepLag := lowReplicationLag.Seconds() + oldRepLag := float64(thc.Stats.SecondsBehindMaster) + newRepLag := float64(newStats.SecondsBehindMaster) + if oldRepLag <= lowRepLag && newRepLag <= lowRepLag { + return true + } + // Skip replag filter when replag remains in the high rep lag range, + // and did not change beyond +/- 10%. + // when there is a high rep lag, it takes a long time for it to reduce, + // so it is not necessary to re-calculate every time. + // In that case, we won't save the new record, so we still + // remember the original replication lag. + if oldRepLag > lowRepLag && newRepLag > lowRepLag && newRepLag < oldRepLag*1.1 && newRepLag > oldRepLag*0.9 { + return true + } + return false +} + // checkConn performs health checking on the given tablet. -func (th *tabletHealthCheck) checkConn(hc *HealthCheck) { - defer hc.connsWG.Done() - defer th.finalizeConn() +func (thc *tabletHealthCheck) checkConn(hc *HealthCheck) { + defer func() { + // TODO(deepthi): We should ensure any return from this func calls the equivalent of hc.deleteTablet + thc.finalizeConn() + hc.connsWG.Done() + }() retryDelay := hc.retryDelay for { - streamCtx, streamCancel := context.WithCancel(th.ctx) + streamCtx, streamCancel := context.WithCancel(thc.ctx) // Setup a watcher that restarts the timer every time an update is received. // If a timeout occurs for a serving tablet, we make it non-serving and send @@ -243,7 +259,7 @@ func (th *tabletHealthCheck) checkConn(hc *HealthCheck) { }() // Read stream health responses. - err := th.stream(streamCtx, func(shr *query.StreamHealthResponse) error { + err := thc.stream(streamCtx, func(shr *query.StreamHealthResponse) error { // We received a message. Reset the back-off. retryDelay = hc.retryDelay // Don't block on send to avoid deadlocks. @@ -251,7 +267,7 @@ func (th *tabletHealthCheck) checkConn(hc *HealthCheck) { case servingStatus <- shr.Serving: default: } - return th.processResponse(hc, shr) + return thc.processResponse(hc, shr) }) // streamCancel to make sure the watcher goroutine terminates. @@ -259,29 +275,26 @@ func (th *tabletHealthCheck) checkConn(hc *HealthCheck) { if err != nil { if strings.Contains(err.Error(), "health stats mismatch") { - hc.deleteTablet(th.Tablet) + hc.deleteTablet(thc.Tablet) return } - res := th.SimpleCopy() + res := thc.SimpleCopy() hc.broadcast(res) } // If there was a timeout send an error. We do this after stream has returned. // This will ensure that this update prevails over any previous message that // stream could have sent. if timedout.Get() { - th.mu.Lock() - th.LastError = fmt.Errorf("healthcheck timed out (latest %v)", th.lastResponseTimestamp) - th.setServingState(false, th.LastError.Error()) - hcErrorCounters.Add([]string{th.Target.Keyspace, th.Target.Shard, topoproto.TabletTypeLString(th.Target.TabletType)}, 1) - res := th.simpleCopyLocked() - th.mu.Unlock() - hc.broadcast(res) + thc.LastError = fmt.Errorf("healthcheck timed out (latest %v)", thc.lastResponseTimestamp) + thc.setServingState(false, thc.LastError.Error()) + hcErrorCounters.Add([]string{thc.Target.Keyspace, thc.Target.Shard, topoproto.TabletTypeLString(thc.Target.TabletType)}, 1) + hc.broadcast(thc.SimpleCopy()) } // Streaming RPC failed e.g. because vttablet was restarted or took too long. // Sleep until the next retry is up or the context is done/canceled. select { - case <-th.ctx.Done(): + case <-thc.ctx.Done(): return case <-time.After(retryDelay): // Exponentially back-off to prevent tight-loop. @@ -294,31 +307,27 @@ func (th *tabletHealthCheck) checkConn(hc *HealthCheck) { } } -func (th *tabletHealthCheck) closeConnection(ctx context.Context, err error) { - th.mu.Lock() - defer th.mu.Unlock() - log.Warningf("tablet %v healthcheck stream error: %v", th.Tablet.Alias, err) - th.setServingState(false, err.Error()) - th.LastError = err - _ = th.Conn.Close(ctx) - th.Conn = nil +func (thc *tabletHealthCheck) closeConnection(ctx context.Context, err error) { + log.Warningf("tablet %v healthcheck stream error: %v", thc.Tablet.Alias, err) + thc.setServingState(false, err.Error()) + thc.LastError = err + _ = thc.Conn.Close(ctx) + thc.Conn = nil } // finalizeConn closes the health checking connection. // To be called only on exit from checkConn(). -func (th *tabletHealthCheck) finalizeConn() { - th.mu.Lock() - defer th.mu.Unlock() - th.setServingState(false, "finalizeConn closing connection") - // Note: checkConn() exits only when th.ctx.Done() is closed. Thus it's +func (thc *tabletHealthCheck) finalizeConn() { + thc.setServingState(false, "finalizeConn closing connection") + // Note: checkConn() exits only when thc.ctx.Done() is closed. Thus it's // safe to simply get Err() value here and assign to LastError. - th.LastError = th.ctx.Err() - if th.Conn != nil { - // Don't use th.ctx because it's already closed. + thc.LastError = thc.ctx.Err() + if thc.Conn != nil { + // Don't use thc.ctx because it's already closed. // Use a separate context, and add a timeout to prevent unbounded waits. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - _ = th.Conn.Close(ctx) - th.Conn = nil + _ = thc.Conn.Close(ctx) + thc.Conn = nil } } diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 4e8433dfddb..2f8e7416c23 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -842,8 +842,8 @@ func (e *Executor) handleShow(ctx context.Context, safeSession *SafeSession, sql case "vitess_tablets": var rows [][]sqltypes.Value if *GatewayImplementation == GatewayImplementationDiscovery { - stats := e.scatterConn.GetLegacyHealthCheckCacheStatus() - for _, s := range stats { + status := e.scatterConn.GetLegacyHealthCheckCacheStatus() + for _, s := range status { for _, ts := range s.TabletsStats { state := "SERVING" if !ts.Serving { @@ -868,8 +868,8 @@ func (e *Executor) handleShow(ctx context.Context, safeSession *SafeSession, sql } } if *GatewayImplementation == tabletGatewayImplementation { - stats := e.scatterConn.GetHealthCheckCacheStatus() - for _, s := range stats { + status := e.scatterConn.GetHealthCheckCacheStatus() + for _, s := range status { for _, ts := range s.TabletsStats { state := "SERVING" if !ts.Serving { From f56e82b95ef1757a913c8401659e0fcd6fda99a0 Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 25 May 2020 15:09:34 -0700 Subject: [PATCH 36/39] healthcheck: make new tabletgateway the default, deprecate discoverygateway Signed-off-by: deepthi --- go/vt/vtgate/executor_framework_test.go | 2 ++ go/vt/vtgate/gateway.go | 2 +- go/vt/vtgate/vtgate_test.go | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index df7cba44c2d..061591fba18 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -342,6 +342,8 @@ const ( ) func createExecutorEnvUsing(t executorType) (executor *Executor, sbc1, sbc2, sbclookup *sandboxconn.SandboxConn) { + // Use legacy gateway until we can rewrite these tests to use new tabletgateway + *GatewayImplementation = GatewayImplementationDiscovery cell := "aa" hc := discovery.NewFakeLegacyHealthCheck() s := createSandbox("TestExecutor") diff --git a/go/vt/vtgate/gateway.go b/go/vt/vtgate/gateway.go index efeef6b87a4..fe7468efe67 100644 --- a/go/vt/vtgate/gateway.go +++ b/go/vt/vtgate/gateway.go @@ -34,7 +34,7 @@ import ( var ( // GatewayImplementation allows you to choose which gateway to use for vtgate routing. Defaults to discoverygateway, other option is tabletgateway - GatewayImplementation = flag.String("gateway_implementation", "discoverygateway", "Allowed values: discoverygateway (default), tabletgateway") + GatewayImplementation = flag.String("gateway_implementation", "tabletgateway", "Allowed values: discoverygateway (deprecated), tabletgateway (default)") initialTabletTimeout = flag.Duration("gateway_initial_tablet_timeout", 30*time.Second, "At startup, the gateway will wait up to that duration to get one tablet per keyspace/shard/tablettype") // RetryCount is the number of times a query will be retried on error // Make this unexported after DiscoveryGateway is deprecated diff --git a/go/vt/vtgate/vtgate_test.go b/go/vt/vtgate/vtgate_test.go index 12befb2aabf..9d0702c4cf3 100644 --- a/go/vt/vtgate/vtgate_test.go +++ b/go/vt/vtgate/vtgate_test.go @@ -71,6 +71,8 @@ func init() { ` hcVTGateTest = discovery.NewFakeLegacyHealthCheck() *transactionMode = "MULTI" + // Use legacy gateway until we can rewrite these tests to use new tabletgateway + *GatewayImplementation = GatewayImplementationDiscovery // The topo.Server is used to start watching the cells described // in '-cells_to_watch' command line parameter, which is // empty by default. So it's unused in this test, set to nil. From a49e0791c77c387ab217a1a6a1f62a9e391dc8d2 Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 25 May 2020 15:51:09 -0700 Subject: [PATCH 37/39] healthcheck: fix vtexplain test to use legacy gateway Signed-off-by: deepthi --- go/vt/vtexplain/vtexplain_flaky_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/vt/vtexplain/vtexplain_flaky_test.go b/go/vt/vtexplain/vtexplain_flaky_test.go index 36b84ba9351..6b22a09c243 100644 --- a/go/vt/vtexplain/vtexplain_flaky_test.go +++ b/go/vt/vtexplain/vtexplain_flaky_test.go @@ -24,6 +24,8 @@ import ( "strings" "testing" + "vitess.io/vitess/go/vt/vtgate" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" ) @@ -40,6 +42,8 @@ func defaultTestOpts() *Options { } func initTest(mode string, opts *Options, t *testing.T) { + // Use legacy gateway until we can rewrite these tests to use new tabletgateway + *vtgate.GatewayImplementation = vtgate.GatewayImplementationDiscovery schema, err := ioutil.ReadFile("testdata/test-schema.sql") require.NoError(t, err) From d9e420b19b56556e022e2045389f90204cb6b1f1 Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 25 May 2020 16:25:19 -0700 Subject: [PATCH 38/39] healthcheck: fix race condition by using thread-safe healthData instead of thread-unsafe healthByAlias from cacheStatusMap Signed-off-by: deepthi --- go/vt/discovery/healthcheck.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index 3994b9b02f0..5b6fceec399 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -473,18 +473,20 @@ func (hc *HealthCheck) cacheStatusMap() map[string]*TabletsCacheStatus { tcsMap := make(map[string]*TabletsCacheStatus) hc.mu.Lock() defer hc.mu.Unlock() - for _, th := range hc.healthByAlias { - key := fmt.Sprintf("%v.%v.%v.%v", th.Tablet.Alias.Cell, th.Target.Keyspace, th.Target.Shard, th.Target.TabletType.String()) - var tcs *TabletsCacheStatus - var ok bool - if tcs, ok = tcsMap[key]; !ok { - tcs = &TabletsCacheStatus{ - Cell: th.Tablet.Alias.Cell, - Target: th.Target, + for _, ths := range hc.healthData { + for _, th := range ths { + key := fmt.Sprintf("%v.%v.%v.%v", th.Tablet.Alias.Cell, th.Target.Keyspace, th.Target.Shard, th.Target.TabletType.String()) + var tcs *TabletsCacheStatus + var ok bool + if tcs, ok = tcsMap[key]; !ok { + tcs = &TabletsCacheStatus{ + Cell: th.Tablet.Alias.Cell, + Target: th.Target, + } + tcsMap[key] = tcs } - tcsMap[key] = tcs + tcs.TabletsStats = append(tcs.TabletsStats, th) } - tcs.TabletsStats = append(tcs.TabletsStats, th.SimpleCopy()) } return tcsMap } From 3343c21ae1143d1cac3d4963d3919fe91cfd3ed5 Mon Sep 17 00:00:00 2001 From: deepthi Date: Mon, 25 May 2020 16:57:24 -0700 Subject: [PATCH 39/39] healthcheck: legacy_replication_lag_algorithm is used by new healthcheck also, so move it to replicationlag.go Signed-off-by: deepthi --- go/vt/discovery/legacy_replicationlag.go | 5 ----- go/vt/discovery/replicationlag.go | 7 ++++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/go/vt/discovery/legacy_replicationlag.go b/go/vt/discovery/legacy_replicationlag.go index 812f7c0dd07..7b21c78ddc3 100644 --- a/go/vt/discovery/legacy_replicationlag.go +++ b/go/vt/discovery/legacy_replicationlag.go @@ -17,15 +17,10 @@ limitations under the License. package discovery import ( - "flag" "fmt" "sort" ) -var ( - legacyReplicationLagAlgorithm = flag.Bool("legacy_replication_lag_algorithm", true, "use the legacy algorithm when selecting the vttablets for serving") -) - // LegacyIsReplicationLagHigh verifies that the given LegacyTabletStats refers to a tablet with high // replication lag, i.e. higher than the configured discovery_low_replication_lag flag. func LegacyIsReplicationLagHigh(tabletStats *LegacyTabletStats) bool { diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index b448d8ece7d..b527bfb8abf 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -25,9 +25,10 @@ import ( var ( // lowReplicationLag defines the duration that replication lag is low enough that the VTTablet is considered healthy. - lowReplicationLag = flag.Duration("discovery_low_replication_lag", 30*time.Second, "the replication lag that is considered low enough to be healthy") - highReplicationLagMinServing = flag.Duration("discovery_high_replication_lag_minimum_serving", 2*time.Hour, "the replication lag that is considered too high when selecting the minimum num vttablets for serving") - minNumTablets = flag.Int("min_number_serving_vttablets", 2, "the minimum number of vttablets that will be continue to be used even with low replication lag") + lowReplicationLag = flag.Duration("discovery_low_replication_lag", 30*time.Second, "the replication lag that is considered low enough to be healthy") + highReplicationLagMinServing = flag.Duration("discovery_high_replication_lag_minimum_serving", 2*time.Hour, "the replication lag that is considered too high when selecting the minimum num vttablets for serving") + minNumTablets = flag.Int("min_number_serving_vttablets", 2, "the minimum number of vttablets that will be continue to be used even with low replication lag") + legacyReplicationLagAlgorithm = flag.Bool("legacy_replication_lag_algorithm", true, "use the legacy algorithm when selecting the vttablets for serving") ) // IsReplicationLagHigh verifies that the given LegacytabletHealth refers to a tablet with high