diff --git a/go/vt/discovery/tablet_stats_cache.go b/go/vt/discovery/tablet_stats_cache.go index 518f20fd07e..d78f3819739 100644 --- a/go/vt/discovery/tablet_stats_cache.go +++ b/go/vt/discovery/tablet_stats_cache.go @@ -22,7 +22,9 @@ import ( log "github.com/golang/glog" querypb "github.com/youtube/vitess/go/vt/proto/query" topodatapb "github.com/youtube/vitess/go/vt/proto/topodata" + "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/topoproto" + "golang.org/x/net/context" ) // TabletStatsCache is a HealthCheckStatsListener that keeps both the @@ -40,10 +42,11 @@ type TabletStatsCache struct { // cell is the cell we are keeping all tablets for. // Note we keep track of all master tablets in all cells. cell string - // mu protects the entries map. It does not protect individual // entries in the map. mu sync.RWMutex + // ts is the topo server in use. + ts *topo.Server // entries maps from keyspace/shard/tabletType to our cache. entries map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry } @@ -65,8 +68,8 @@ type tabletStatsCacheEntry struct { // Note we do the registration in this code to guarantee we call // SetListener with sendDownEvents=true, as we need these events // to maintain the integrity of our cache. -func NewTabletStatsCache(hc HealthCheck, cell string) *TabletStatsCache { - return newTabletStatsCache(hc, cell, true /* setListener */) +func NewTabletStatsCache(hc HealthCheck, ts *topo.Server, cell string) *TabletStatsCache { + return newTabletStatsCache(hc, ts, cell, true /* setListener */) } // NewTabletStatsCacheDoNotSetListener is identical to NewTabletStatsCache @@ -76,13 +79,14 @@ func NewTabletStatsCache(hc HealthCheck, cell string) *TabletStatsCache { // When the caller sets its own listener on "hc", they must make sure that they // set the parameter "sendDownEvents" to "true" or this cache won't properly // remove tablets whose tablet type changes. -func NewTabletStatsCacheDoNotSetListener(cell string) *TabletStatsCache { - return newTabletStatsCache(nil, cell, false /* setListener */) +func NewTabletStatsCacheDoNotSetListener(ts *topo.Server, cell string) *TabletStatsCache { + return newTabletStatsCache(nil, ts, cell, false /* setListener */) } -func newTabletStatsCache(hc HealthCheck, cell string, setListener bool) *TabletStatsCache { +func newTabletStatsCache(hc HealthCheck, ts *topo.Server, cell string, setListener bool) *TabletStatsCache { tc := &TabletStatsCache{ cell: cell, + ts: ts, entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry), } @@ -142,10 +146,14 @@ func (tc *TabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStat return e } +func (tc *TabletStatsCache) getRegionByCell(cell string) string { + return topo.GetRegionByCell(context.Background(), tc.ts, cell) +} + // StatsUpdate is part of the HealthCheckStatsListener interface. func (tc *TabletStatsCache) StatsUpdate(ts *TabletStats) { - if ts.Target.TabletType != topodatapb.TabletType_MASTER && ts.Tablet.Alias.Cell != tc.cell { - // this is for a non-master tablet in a different cell, drop it + if ts.Target.TabletType != topodatapb.TabletType_MASTER && ts.Tablet.Alias.Cell != tc.cell && tc.getRegionByCell(ts.Tablet.Alias.Cell) != tc.getRegionByCell(tc.cell) { + // this is for a non-master tablet in a different cell and a different region, drop it return } diff --git a/go/vt/discovery/tablet_stats_cache_test.go b/go/vt/discovery/tablet_stats_cache_test.go index 1e132634155..4da8493e9f3 100644 --- a/go/vt/discovery/tablet_stats_cache_test.go +++ b/go/vt/discovery/tablet_stats_cache_test.go @@ -27,6 +27,13 @@ import ( // TestTabletStatsCache tests the functionality of the TabletStatsCache class. func TestTabletStatsCache(t *testing.T) { + defer topo.UpdateCellsToRegionsForTests(map[string]string{}) + topo.UpdateCellsToRegionsForTests(map[string]string{ + "cell": "region1", + "cell1": "region1", + "cell2": "region2", + }) + // We want to unit test TabletStatsCache without a full-blown // HealthCheck object, so we can't call NewTabletStatsCache. // So we just construct this object here. @@ -207,4 +214,46 @@ func TestTabletStatsCache(t *testing.T) { if len(a) != 1 || !ts1.DeepEqual(&a[0]) { t.Errorf("unexpected result: %v", a) } + + // add a third tablet as slave in diff cell, same region + tablet3 := topo.NewTablet(12, "cell1", "host3") + ts3 := &TabletStats{ + Key: "t3", + Tablet: tablet3, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Up: true, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, + } + tsc.StatsUpdate(ts3) + // check it's there + a = tsc.GetTabletStats("k", "s", topodatapb.TabletType_REPLICA) + if len(a) != 1 { + t.Errorf("unexpected result: %v", a) + } + a = tsc.GetHealthyTabletStats("k", "s", topodatapb.TabletType_REPLICA) + if len(a) != 1 { + t.Errorf("unexpected result: %v", a) + } + + // add a 4th slave tablet in a diff cell, diff region + tablet4 := topo.NewTablet(13, "cell2", "host4") + ts4 := &TabletStats{ + Key: "t4", + Tablet: tablet4, + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Up: true, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2}, + } + tsc.StatsUpdate(ts4) + // check it's *NOT* there + a = tsc.GetTabletStats("k", "s", topodatapb.TabletType_REPLICA) + if len(a) != 1 { + t.Errorf("unexpected result: %v", a) + } + a = tsc.GetHealthyTabletStats("k", "s", topodatapb.TabletType_REPLICA) + if len(a) != 1 { + t.Errorf("unexpected result: %v", a) + } } diff --git a/go/vt/discovery/tablet_stats_cache_wait_test.go b/go/vt/discovery/tablet_stats_cache_wait_test.go index c89f52f2c2c..ece0af3a188 100644 --- a/go/vt/discovery/tablet_stats_cache_wait_test.go +++ b/go/vt/discovery/tablet_stats_cache_wait_test.go @@ -131,7 +131,7 @@ func TestWaitForTablets(t *testing.T) { createFakeConn(tablet, input) hc := NewHealthCheck(1*time.Millisecond, 1*time.Hour) - tsc := NewTabletStatsCache(hc, "cell") + tsc := NewTabletStatsCache(hc, nil, "cell") hc.AddTablet(tablet, "") // this should time out diff --git a/go/vt/proto/topodata/topodata.pb.go b/go/vt/proto/topodata/topodata.pb.go index 86475d3f5db..784a1b3b0bb 100644 --- a/go/vt/proto/topodata/topodata.pb.go +++ b/go/vt/proto/topodata/topodata.pb.go @@ -755,6 +755,8 @@ type CellInfo struct { // Root is the path to store data in. It is only used when talking // to server_address. Root string `protobuf:"bytes,2,opt,name=root" json:"root,omitempty"` + // Region is a group this cell belongs to + Region string `protobuf:"bytes,3,opt,name=region" json:"region,omitempty"` } func (m *CellInfo) Reset() { *m = CellInfo{} } @@ -776,6 +778,13 @@ func (m *CellInfo) GetRoot() string { return "" } +func (m *CellInfo) GetRegion() string { + if m != nil { + return m.Region + } + return "" +} + func init() { proto.RegisterType((*KeyRange)(nil), "topodata.KeyRange") proto.RegisterType((*TabletAlias)(nil), "topodata.TabletAlias") @@ -800,75 +809,76 @@ func init() { func init() { proto.RegisterFile("topodata.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1115 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5f, 0x6f, 0xe2, 0x46, - 0x10, 0xaf, 0xc1, 0x10, 0x18, 0x03, 0xe7, 0x6c, 0x73, 0x95, 0xe5, 0xea, 0x54, 0x84, 0x54, 0x15, - 0x5d, 0x55, 0x5a, 0x71, 0xbd, 0x36, 0x3a, 0xa9, 0x52, 0x08, 0xf1, 0xf5, 0xc8, 0x1f, 0x42, 0x17, - 0xa2, 0x36, 0x4f, 0x96, 0x83, 0x37, 0x39, 0x2b, 0xc6, 0xf6, 0xed, 0x2e, 0x91, 0xf8, 0x0c, 0xf7, - 0xd0, 0x7b, 0xee, 0x37, 0xe9, 0x53, 0x1f, 0xfb, 0xb5, 0xaa, 0xdd, 0xb5, 0xc1, 0x90, 0x26, 0xcd, - 0x55, 0x79, 0xca, 0xcc, 0xce, 0x1f, 0xcf, 0xfc, 0xe6, 0x37, 0x13, 0xa0, 0xc1, 0xe3, 0x24, 0xf6, - 0x3d, 0xee, 0x75, 0x12, 0x1a, 0xf3, 0x18, 0x55, 0x32, 0xbd, 0xd5, 0x85, 0xca, 0x11, 0x59, 0x60, - 0x2f, 0xba, 0x22, 0x68, 0x07, 0x4a, 0x8c, 0x7b, 0x94, 0x5b, 0x5a, 0x53, 0x6b, 0xd7, 0xb0, 0x52, - 0x90, 0x09, 0x45, 0x12, 0xf9, 0x56, 0x41, 0xbe, 0x09, 0xb1, 0xf5, 0x02, 0x8c, 0x89, 0x77, 0x11, - 0x12, 0xde, 0x0b, 0x03, 0x8f, 0x21, 0x04, 0xfa, 0x94, 0x84, 0xa1, 0x8c, 0xaa, 0x62, 0x29, 0x8b, - 0xa0, 0x79, 0xa0, 0x82, 0xea, 0x58, 0x88, 0xad, 0x3f, 0x75, 0x28, 0xab, 0x28, 0xf4, 0x35, 0x94, - 0x3c, 0x11, 0x29, 0x23, 0x8c, 0xee, 0xd3, 0xce, 0xb2, 0xba, 0x5c, 0x5a, 0xac, 0x7c, 0x90, 0x0d, - 0x95, 0xb7, 0x31, 0xe3, 0x91, 0x37, 0x23, 0x32, 0x5d, 0x15, 0x2f, 0x75, 0xb4, 0x0b, 0x95, 0x24, - 0xa6, 0xdc, 0x9d, 0x79, 0x89, 0xa5, 0x37, 0x8b, 0x6d, 0xa3, 0xfb, 0x6c, 0x33, 0x57, 0x67, 0x14, - 0x53, 0x7e, 0xe2, 0x25, 0x4e, 0xc4, 0xe9, 0x02, 0x6f, 0x25, 0x4a, 0x13, 0x59, 0xaf, 0xc9, 0x82, - 0x25, 0xde, 0x94, 0x58, 0x25, 0x95, 0x35, 0xd3, 0x25, 0x0c, 0x6f, 0x3d, 0xea, 0x5b, 0x65, 0x69, - 0x50, 0x0a, 0xfa, 0x16, 0xaa, 0xd7, 0x64, 0xe1, 0x52, 0x81, 0x94, 0xb5, 0x25, 0x0b, 0x47, 0xab, - 0x8f, 0x65, 0x18, 0xca, 0x34, 0x0a, 0xcd, 0x36, 0xe8, 0x7c, 0x91, 0x10, 0xab, 0xd2, 0xd4, 0xda, - 0x8d, 0xee, 0xce, 0x66, 0x61, 0x93, 0x45, 0x42, 0xb0, 0xf4, 0x40, 0x6d, 0x30, 0xfd, 0x0b, 0x57, - 0x74, 0xe4, 0xc6, 0x37, 0x84, 0xd2, 0xc0, 0x27, 0x56, 0x55, 0x7e, 0xbb, 0xe1, 0x5f, 0x0c, 0xbd, - 0x19, 0x39, 0x4d, 0x5f, 0x51, 0x07, 0x74, 0xee, 0x5d, 0x31, 0x0b, 0x64, 0xb3, 0xf6, 0xad, 0x66, - 0x27, 0xde, 0x15, 0x53, 0x9d, 0x4a, 0x3f, 0xf4, 0x25, 0x34, 0x66, 0x0b, 0xf6, 0x2e, 0x74, 0x97, - 0x10, 0xd6, 0x64, 0xde, 0xba, 0x7c, 0x7d, 0x93, 0xe1, 0xf8, 0x0c, 0x40, 0xb9, 0x09, 0x78, 0xac, - 0x7a, 0x53, 0x6b, 0x97, 0x70, 0x55, 0xbe, 0x08, 0xf4, 0xec, 0x57, 0x50, 0xcb, 0xa3, 0x28, 0x86, - 0x7b, 0x4d, 0x16, 0xe9, 0xbc, 0x85, 0x28, 0x20, 0xbb, 0xf1, 0xc2, 0xb9, 0x9a, 0x50, 0x09, 0x2b, - 0xe5, 0x55, 0x61, 0x57, 0xb3, 0x7f, 0x84, 0xea, 0xb2, 0xa8, 0xff, 0x0a, 0xac, 0xe6, 0x02, 0x0f, - 0xf5, 0x4a, 0xd1, 0xd4, 0x0f, 0xf5, 0x8a, 0x61, 0xd6, 0x5a, 0xef, 0xcb, 0x50, 0x1a, 0xcb, 0x29, - 0xec, 0x42, 0x6d, 0xe6, 0x31, 0x4e, 0xa8, 0xfb, 0x00, 0x06, 0x19, 0xca, 0x55, 0xb1, 0x74, 0x6d, - 0x7e, 0x85, 0x07, 0xcc, 0xef, 0x27, 0xa8, 0x31, 0x42, 0x6f, 0x88, 0xef, 0x8a, 0x21, 0x31, 0xab, - 0xb8, 0x89, 0xb9, 0xac, 0xa8, 0x33, 0x96, 0x3e, 0x72, 0x9a, 0x06, 0x5b, 0xca, 0x0c, 0xed, 0x41, - 0x9d, 0xc5, 0x73, 0x3a, 0x25, 0xae, 0xe4, 0x0f, 0x4b, 0x09, 0xfa, 0xf9, 0xad, 0x78, 0xe9, 0x24, - 0x65, 0x5c, 0x63, 0x2b, 0x85, 0x09, 0x6c, 0xc4, 0x2e, 0x31, 0xab, 0xd4, 0x2c, 0x0a, 0x6c, 0xa4, - 0x82, 0x5e, 0xc3, 0x13, 0x2e, 0x7b, 0x74, 0xa7, 0x71, 0xc4, 0x69, 0x1c, 0x32, 0xab, 0xbc, 0x49, - 0x7d, 0x95, 0x59, 0x41, 0xd1, 0x57, 0x5e, 0xb8, 0xc1, 0xf3, 0x2a, 0xb3, 0xcf, 0x01, 0x56, 0xa5, - 0xa3, 0x97, 0x60, 0xa4, 0x59, 0x25, 0x67, 0xb5, 0x7b, 0x38, 0x0b, 0x7c, 0x29, 0xaf, 0x4a, 0x2c, - 0xe4, 0x4a, 0xb4, 0xff, 0xd0, 0xc0, 0xc8, 0xb5, 0x95, 0x1d, 0x03, 0x6d, 0x79, 0x0c, 0xd6, 0xd6, - 0xaf, 0x70, 0xd7, 0xfa, 0x15, 0xef, 0x5c, 0x3f, 0xfd, 0x01, 0xe3, 0xfb, 0x0c, 0xca, 0xb2, 0xd0, - 0x0c, 0xbe, 0x54, 0xb3, 0xff, 0xd2, 0xa0, 0xbe, 0x86, 0xcc, 0xa3, 0xf6, 0x8e, 0xba, 0xf0, 0xd4, - 0x0f, 0x98, 0xf0, 0x72, 0xdf, 0xcd, 0x09, 0x5d, 0xb8, 0x82, 0x13, 0xc1, 0x94, 0xc8, 0x6e, 0x2a, - 0xf8, 0xd3, 0xd4, 0xf8, 0x8b, 0xb0, 0x8d, 0x95, 0x09, 0x7d, 0x03, 0xe8, 0x22, 0xf4, 0xa6, 0xd7, - 0x61, 0xc0, 0xb8, 0xa0, 0x9b, 0x2a, 0x5b, 0x97, 0x69, 0xb7, 0x73, 0x16, 0x59, 0x08, 0x6b, 0xfd, - 0x5d, 0x90, 0x37, 0x5b, 0xa1, 0xf5, 0x1d, 0xec, 0x48, 0x80, 0x82, 0xe8, 0xca, 0x9d, 0xc6, 0xe1, - 0x7c, 0x16, 0xc9, 0x43, 0x92, 0xee, 0x18, 0xca, 0x6c, 0x7d, 0x69, 0x12, 0xb7, 0x04, 0x1d, 0xde, - 0x8e, 0x90, 0x7d, 0x17, 0x64, 0xdf, 0xd6, 0x1a, 0xa8, 0xf2, 0x1b, 0x03, 0xc5, 0xee, 0x8d, 0x5c, - 0x12, 0x83, 0xbd, 0xe5, 0x8e, 0x5c, 0xd2, 0x78, 0xc6, 0x6e, 0x1f, 0xe1, 0x2c, 0x47, 0xba, 0x26, - 0xaf, 0x69, 0x3c, 0xcb, 0xd6, 0x44, 0xc8, 0xcc, 0x9e, 0x67, 0x34, 0x14, 0xea, 0xe3, 0x8e, 0x22, - 0x4f, 0xb2, 0xe2, 0x3a, 0xc9, 0xd4, 0x75, 0x69, 0xbd, 0xd7, 0xc0, 0x54, 0x9b, 0x47, 0x92, 0x30, - 0x98, 0x7a, 0x3c, 0x88, 0x23, 0xf4, 0x12, 0x4a, 0x51, 0xec, 0x13, 0x71, 0x5b, 0x44, 0x33, 0x5f, - 0x6c, 0xac, 0x55, 0xce, 0xb5, 0x33, 0x8c, 0x7d, 0x82, 0x95, 0xb7, 0xbd, 0x07, 0xba, 0x50, 0xc5, - 0x85, 0x4a, 0x5b, 0x78, 0xc8, 0x85, 0xe2, 0x2b, 0xa5, 0x75, 0x06, 0x8d, 0xf4, 0x0b, 0x97, 0x84, - 0x92, 0x68, 0x4a, 0xc4, 0x7f, 0xd6, 0xdc, 0x30, 0xa5, 0xfc, 0xd1, 0x77, 0xac, 0xf5, 0x41, 0x07, - 0x63, 0x4c, 0x6f, 0x96, 0x8c, 0xf9, 0x19, 0x20, 0xf1, 0x28, 0x0f, 0x44, 0x07, 0x59, 0x93, 0x5f, - 0xe5, 0x9a, 0x5c, 0xb9, 0x2e, 0xa7, 0x37, 0xca, 0xfc, 0x71, 0x2e, 0xf4, 0x4e, 0xea, 0x15, 0x3e, - 0x9a, 0x7a, 0xc5, 0xff, 0x41, 0xbd, 0x1e, 0x18, 0x39, 0xea, 0xa5, 0xcc, 0x6b, 0xfe, 0x7b, 0x1f, - 0x39, 0xf2, 0xc1, 0x8a, 0x7c, 0xf6, 0xef, 0x1a, 0x6c, 0xdf, 0x6a, 0x51, 0x70, 0x30, 0x77, 0xf7, - 0xef, 0xe7, 0xe0, 0xea, 0xe0, 0xa3, 0x3e, 0x98, 0xb2, 0x4a, 0x97, 0x66, 0xe3, 0x53, 0x74, 0x34, - 0xf2, 0x7d, 0xad, 0xcf, 0x17, 0x3f, 0x61, 0x6b, 0x3a, 0xb3, 0xdd, 0xc7, 0xd8, 0x86, 0x7b, 0x8e, - 0xeb, 0xa1, 0x5e, 0x29, 0x99, 0xe5, 0x96, 0x03, 0x95, 0x3e, 0x09, 0xc3, 0x41, 0x74, 0x19, 0x8b, - 0x9f, 0x08, 0xb2, 0x0b, 0xea, 0x7a, 0xbe, 0x4f, 0x09, 0x63, 0x29, 0xdb, 0xea, 0xea, 0xb5, 0xa7, - 0x1e, 0x05, 0x15, 0x69, 0x1c, 0xf3, 0x34, 0xa1, 0x94, 0x9f, 0x77, 0xa1, 0xb1, 0x3e, 0x28, 0x54, - 0x85, 0xd2, 0xd9, 0x70, 0xec, 0x4c, 0xcc, 0x4f, 0x10, 0x40, 0xf9, 0x6c, 0x30, 0x9c, 0xfc, 0xf0, - 0xbd, 0xa9, 0x89, 0xe7, 0xfd, 0xf3, 0x89, 0x33, 0x36, 0x0b, 0xcf, 0x3f, 0x68, 0x00, 0xab, 0xba, - 0x91, 0x01, 0x5b, 0x67, 0xc3, 0xa3, 0xe1, 0xe9, 0xaf, 0x43, 0x15, 0x72, 0xd2, 0x1b, 0x4f, 0x1c, - 0x6c, 0x6a, 0xc2, 0x80, 0x9d, 0xd1, 0xf1, 0xa0, 0xdf, 0x33, 0x0b, 0xc2, 0x80, 0x0f, 0x4e, 0x87, - 0xc7, 0xe7, 0x66, 0x51, 0xe6, 0xea, 0x4d, 0xfa, 0x6f, 0x94, 0x38, 0x1e, 0xf5, 0xb0, 0x63, 0xea, - 0xc8, 0x84, 0x9a, 0xf3, 0xdb, 0xc8, 0xc1, 0x83, 0x13, 0x67, 0x38, 0xe9, 0x1d, 0x9b, 0x25, 0x11, - 0xb3, 0xdf, 0xeb, 0x1f, 0x9d, 0x8d, 0xcc, 0xb2, 0x4a, 0x36, 0x9e, 0x9c, 0x62, 0xc7, 0xdc, 0x12, - 0xca, 0x01, 0xee, 0x0d, 0x86, 0xce, 0x81, 0x59, 0xb1, 0x0b, 0xa6, 0xb6, 0xbf, 0x0d, 0x4f, 0x82, - 0xb8, 0x73, 0x13, 0x70, 0xc2, 0x98, 0xfa, 0x7d, 0x7c, 0x51, 0x96, 0x7f, 0x5e, 0xfc, 0x13, 0x00, - 0x00, 0xff, 0xff, 0xba, 0xda, 0xa7, 0xb1, 0x38, 0x0b, 0x00, 0x00, + // 1129 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdf, 0x6f, 0xe2, 0xc6, + 0x13, 0xff, 0x1a, 0x0c, 0x81, 0x31, 0x70, 0xce, 0x7e, 0x73, 0x95, 0xe5, 0xea, 0x54, 0x84, 0x54, + 0x15, 0x5d, 0x55, 0x5a, 0x71, 0xbd, 0x36, 0x3a, 0xa9, 0x52, 0x08, 0xe1, 0x7a, 0xe4, 0x07, 0xa1, + 0x0b, 0x51, 0x9b, 0x87, 0xca, 0x72, 0xf0, 0x26, 0x67, 0xc5, 0x78, 0xb9, 0xdd, 0x25, 0x12, 0x7f, + 0xc3, 0x3d, 0xf4, 0x9e, 0xfb, 0x9f, 0xf4, 0xa9, 0x8f, 0xfd, 0xb7, 0xaa, 0xdd, 0xb5, 0xc1, 0x90, + 0x26, 0xcd, 0x55, 0x79, 0xca, 0xce, 0xee, 0xcc, 0x78, 0x3e, 0x9f, 0xf9, 0xcc, 0x04, 0xa8, 0x09, + 0x3a, 0xa3, 0x81, 0x2f, 0xfc, 0xd6, 0x8c, 0x51, 0x41, 0x51, 0x29, 0xb5, 0x1b, 0x6d, 0x28, 0x1d, + 0x91, 0x05, 0xf6, 0xe3, 0x2b, 0x82, 0x76, 0xa0, 0xc0, 0x85, 0xcf, 0x84, 0x63, 0xd4, 0x8d, 0x66, + 0x05, 0x6b, 0x03, 0xd9, 0x90, 0x27, 0x71, 0xe0, 0xe4, 0xd4, 0x9d, 0x3c, 0x36, 0x5e, 0x80, 0x35, + 0xf6, 0x2f, 0x22, 0x22, 0x3a, 0x51, 0xe8, 0x73, 0x84, 0xc0, 0x9c, 0x90, 0x28, 0x52, 0x51, 0x65, + 0xac, 0xce, 0x32, 0x68, 0x1e, 0xea, 0xa0, 0x2a, 0x96, 0xc7, 0xc6, 0x1f, 0x26, 0x14, 0x75, 0x14, + 0xfa, 0x12, 0x0a, 0xbe, 0x8c, 0x54, 0x11, 0x56, 0xfb, 0x69, 0x6b, 0x59, 0x5d, 0x26, 0x2d, 0xd6, + 0x3e, 0xc8, 0x85, 0xd2, 0x5b, 0xca, 0x45, 0xec, 0x4f, 0x89, 0x4a, 0x57, 0xc6, 0x4b, 0x1b, 0xed, + 0x42, 0x69, 0x46, 0x99, 0xf0, 0xa6, 0xfe, 0xcc, 0x31, 0xeb, 0xf9, 0xa6, 0xd5, 0x7e, 0xb6, 0x99, + 0xab, 0x35, 0xa4, 0x4c, 0x9c, 0xf8, 0xb3, 0x5e, 0x2c, 0xd8, 0x02, 0x6f, 0xcd, 0xb4, 0x25, 0xb3, + 0x5e, 0x93, 0x05, 0x9f, 0xf9, 0x13, 0xe2, 0x14, 0x74, 0xd6, 0xd4, 0x56, 0x34, 0xbc, 0xf5, 0x59, + 0xe0, 0x14, 0xd5, 0x83, 0x36, 0xd0, 0xd7, 0x50, 0xbe, 0x26, 0x0b, 0x8f, 0x49, 0xa6, 0x9c, 0x2d, + 0x55, 0x38, 0x5a, 0x7d, 0x2c, 0xe5, 0x50, 0xa5, 0xd1, 0x6c, 0x36, 0xc1, 0x14, 0x8b, 0x19, 0x71, + 0x4a, 0x75, 0xa3, 0x59, 0x6b, 0xef, 0x6c, 0x16, 0x36, 0x5e, 0xcc, 0x08, 0x56, 0x1e, 0xa8, 0x09, + 0x76, 0x70, 0xe1, 0x49, 0x44, 0x1e, 0xbd, 0x21, 0x8c, 0x85, 0x01, 0x71, 0xca, 0xea, 0xdb, 0xb5, + 0xe0, 0x62, 0xe0, 0x4f, 0xc9, 0x69, 0x72, 0x8b, 0x5a, 0x60, 0x0a, 0xff, 0x8a, 0x3b, 0xa0, 0xc0, + 0xba, 0xb7, 0xc0, 0x8e, 0xfd, 0x2b, 0xae, 0x91, 0x2a, 0x3f, 0xf4, 0x39, 0xd4, 0xa6, 0x0b, 0xfe, + 0x2e, 0xf2, 0x96, 0x14, 0x56, 0x54, 0xde, 0xaa, 0xba, 0x7d, 0x93, 0xf2, 0xf8, 0x0c, 0x40, 0xbb, + 0x49, 0x7a, 0x9c, 0x6a, 0xdd, 0x68, 0x16, 0x70, 0x59, 0xdd, 0x48, 0xf6, 0xdc, 0x57, 0x50, 0xc9, + 0xb2, 0x28, 0x9b, 0x7b, 0x4d, 0x16, 0x49, 0xbf, 0xe5, 0x51, 0x52, 0x76, 0xe3, 0x47, 0x73, 0xdd, + 0xa1, 0x02, 0xd6, 0xc6, 0xab, 0xdc, 0xae, 0xe1, 0x7e, 0x0f, 0xe5, 0x65, 0x51, 0xff, 0x16, 0x58, + 0xce, 0x04, 0x1e, 0x9a, 0xa5, 0xbc, 0x6d, 0x1e, 0x9a, 0x25, 0xcb, 0xae, 0x34, 0xde, 0x17, 0xa1, + 0x30, 0x52, 0x5d, 0xd8, 0x85, 0xca, 0xd4, 0xe7, 0x82, 0x30, 0xef, 0x01, 0x0a, 0xb2, 0xb4, 0xab, + 0x56, 0xe9, 0x5a, 0xff, 0x72, 0x0f, 0xe8, 0xdf, 0x0f, 0x50, 0xe1, 0x84, 0xdd, 0x90, 0xc0, 0x93, + 0x4d, 0xe2, 0x4e, 0x7e, 0x93, 0x73, 0x55, 0x51, 0x6b, 0xa4, 0x7c, 0x54, 0x37, 0x2d, 0xbe, 0x3c, + 0x73, 0xb4, 0x07, 0x55, 0x4e, 0xe7, 0x6c, 0x42, 0x3c, 0xa5, 0x1f, 0x9e, 0x08, 0xf4, 0xd3, 0x5b, + 0xf1, 0xca, 0x49, 0x9d, 0x71, 0x85, 0xaf, 0x0c, 0x2e, 0xb9, 0x91, 0xb3, 0xc4, 0x9d, 0x42, 0x3d, + 0x2f, 0xb9, 0x51, 0x06, 0x7a, 0x0d, 0x4f, 0x84, 0xc2, 0xe8, 0x4d, 0x68, 0x2c, 0x18, 0x8d, 0xb8, + 0x53, 0xdc, 0x94, 0xbe, 0xce, 0xac, 0xa9, 0xe8, 0x6a, 0x2f, 0x5c, 0x13, 0x59, 0x93, 0xbb, 0xe7, + 0x00, 0xab, 0xd2, 0xd1, 0x4b, 0xb0, 0x92, 0xac, 0x4a, 0xb3, 0xc6, 0x3d, 0x9a, 0x05, 0xb1, 0x3c, + 0xaf, 0x4a, 0xcc, 0x65, 0x4a, 0x74, 0x7f, 0x37, 0xc0, 0xca, 0xc0, 0x4a, 0x97, 0x81, 0xb1, 0x5c, + 0x06, 0x6b, 0xe3, 0x97, 0xbb, 0x6b, 0xfc, 0xf2, 0x77, 0x8e, 0x9f, 0xf9, 0x80, 0xf6, 0x7d, 0x02, + 0x45, 0x55, 0x68, 0x4a, 0x5f, 0x62, 0xb9, 0x7f, 0x1a, 0x50, 0x5d, 0x63, 0xe6, 0x51, 0xb1, 0xa3, + 0x36, 0x3c, 0x0d, 0x42, 0x2e, 0xbd, 0xbc, 0x77, 0x73, 0xc2, 0x16, 0x9e, 0xd4, 0x44, 0x38, 0x21, + 0x0a, 0x4d, 0x09, 0xff, 0x3f, 0x79, 0xfc, 0x49, 0xbe, 0x8d, 0xf4, 0x13, 0xfa, 0x0a, 0xd0, 0x45, + 0xe4, 0x4f, 0xae, 0xa3, 0x90, 0x0b, 0x29, 0x37, 0x5d, 0xb6, 0xa9, 0xd2, 0x6e, 0x67, 0x5e, 0x54, + 0x21, 0xbc, 0xf1, 0x57, 0x4e, 0xed, 0x6c, 0xcd, 0xd6, 0x37, 0xb0, 0xa3, 0x08, 0x0a, 0xe3, 0x2b, + 0x6f, 0x42, 0xa3, 0xf9, 0x34, 0x56, 0x8b, 0x24, 0x99, 0x31, 0x94, 0xbe, 0x75, 0xd5, 0x93, 0xdc, + 0x25, 0xe8, 0xf0, 0x76, 0x84, 0xc2, 0x9d, 0x53, 0xb8, 0x9d, 0x35, 0x52, 0xd5, 0x37, 0xfa, 0x5a, + 0xdd, 0x1b, 0xb9, 0x14, 0x07, 0x7b, 0xcb, 0x19, 0xb9, 0x64, 0x74, 0xca, 0x6f, 0x2f, 0xe1, 0x34, + 0x47, 0x32, 0x26, 0xaf, 0x19, 0x9d, 0xa6, 0x63, 0x22, 0xcf, 0xdc, 0x9d, 0xa7, 0x32, 0x94, 0xe6, + 0xe3, 0xb6, 0x22, 0x2b, 0xb2, 0xfc, 0xba, 0xc8, 0xf4, 0x76, 0x69, 0xbc, 0x37, 0xc0, 0xd6, 0x93, + 0x47, 0x66, 0x51, 0x38, 0xf1, 0x45, 0x48, 0x63, 0xf4, 0x12, 0x0a, 0x31, 0x0d, 0x88, 0xdc, 0x2d, + 0x12, 0xcc, 0x67, 0x1b, 0x63, 0x95, 0x71, 0x6d, 0x0d, 0x68, 0x40, 0xb0, 0xf6, 0x76, 0xf7, 0xc0, + 0x94, 0xa6, 0xdc, 0x50, 0x09, 0x84, 0x87, 0x6c, 0x28, 0xb1, 0x32, 0x1a, 0x67, 0x50, 0x4b, 0xbe, + 0x70, 0x49, 0x18, 0x89, 0x27, 0x44, 0xfe, 0x67, 0xcd, 0x34, 0x53, 0x9d, 0x3f, 0x7a, 0x8f, 0x35, + 0x3e, 0x98, 0x60, 0x8d, 0xd8, 0xcd, 0x52, 0x31, 0x3f, 0x02, 0xcc, 0x7c, 0x26, 0x42, 0x89, 0x20, + 0x05, 0xf9, 0x45, 0x06, 0xe4, 0xca, 0x75, 0xd9, 0xbd, 0x61, 0xea, 0x8f, 0x33, 0xa1, 0x77, 0x4a, + 0x2f, 0xf7, 0xd1, 0xd2, 0xcb, 0xff, 0x07, 0xe9, 0x75, 0xc0, 0xca, 0x48, 0x2f, 0x51, 0x5e, 0xfd, + 0x9f, 0x71, 0x64, 0xc4, 0x07, 0x2b, 0xf1, 0xb9, 0xbf, 0x19, 0xb0, 0x7d, 0x0b, 0xa2, 0xd4, 0x60, + 0x66, 0xef, 0xdf, 0xaf, 0xc1, 0xd5, 0xc2, 0x47, 0x5d, 0xb0, 0x55, 0x95, 0x1e, 0x4b, 0xdb, 0xa7, + 0xe5, 0x68, 0x65, 0x71, 0xad, 0xf7, 0x17, 0x3f, 0xe1, 0x6b, 0x36, 0x77, 0xbd, 0xc7, 0x98, 0x86, + 0x7b, 0x96, 0xeb, 0xa1, 0x59, 0x2a, 0xd8, 0xc5, 0xc6, 0xaf, 0x50, 0xea, 0x92, 0x28, 0xea, 0xc7, + 0x97, 0x54, 0xfe, 0x44, 0x50, 0x28, 0x98, 0xe7, 0x07, 0x01, 0x23, 0x9c, 0x27, 0x6a, 0xab, 0xea, + 0xdb, 0x8e, 0xbe, 0x94, 0x52, 0x64, 0x94, 0x8a, 0x24, 0xa1, 0x3a, 0xcb, 0x15, 0xcb, 0xc8, 0x55, + 0x48, 0xe3, 0x64, 0xbc, 0x12, 0xeb, 0x79, 0x1b, 0x6a, 0xeb, 0x0d, 0x44, 0x65, 0x28, 0x9c, 0x0d, + 0x46, 0xbd, 0xb1, 0xfd, 0x3f, 0x04, 0x50, 0x3c, 0xeb, 0x0f, 0xc6, 0xdf, 0x7d, 0x6b, 0x1b, 0xf2, + 0x7a, 0xff, 0x7c, 0xdc, 0x1b, 0xd9, 0xb9, 0xe7, 0x1f, 0x0c, 0x80, 0x15, 0x1e, 0x64, 0xc1, 0xd6, + 0xd9, 0xe0, 0x68, 0x70, 0xfa, 0xf3, 0x40, 0x87, 0x9c, 0x74, 0x46, 0xe3, 0x1e, 0xb6, 0x0d, 0xf9, + 0x80, 0x7b, 0xc3, 0xe3, 0x7e, 0xb7, 0x63, 0xe7, 0xe4, 0x03, 0x3e, 0x38, 0x1d, 0x1c, 0x9f, 0xdb, + 0x79, 0x95, 0xab, 0x33, 0xee, 0xbe, 0xd1, 0xc7, 0xd1, 0xb0, 0x83, 0x7b, 0xb6, 0x89, 0x6c, 0xa8, + 0xf4, 0x7e, 0x19, 0xf6, 0x70, 0xff, 0xa4, 0x37, 0x18, 0x77, 0x8e, 0xed, 0x82, 0x8c, 0xd9, 0xef, + 0x74, 0x8f, 0xce, 0x86, 0x76, 0x51, 0x27, 0x1b, 0x8d, 0x4f, 0x71, 0xcf, 0xde, 0x92, 0xc6, 0x01, + 0xee, 0xf4, 0x07, 0xbd, 0x03, 0xbb, 0xe4, 0xe6, 0x6c, 0x63, 0x7f, 0x1b, 0x9e, 0x84, 0xb4, 0x75, + 0x13, 0x0a, 0xc2, 0xb9, 0xfe, 0xdd, 0x7c, 0x51, 0x54, 0x7f, 0x5e, 0xfc, 0x1d, 0x00, 0x00, 0xff, + 0xff, 0xc7, 0x53, 0xa5, 0x90, 0x50, 0x0b, 0x00, 0x00, } diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index e903acb9b7c..f48fc82f7fb 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -178,6 +178,12 @@ type SrvTopoServer interface { WatchSrvVSchema(ctx context.Context, cell string) (*WatchSrvVSchemaData, <-chan *WatchSrvVSchemaData, CancelFunc) } +type cellsToRegionsMap struct { + mu sync.Mutex + // cellsToRegions contains all cell->region mappings + cellsToRegions map[string]string +} + var ( // topoImplementation is the flag for which implementation to use. topoImplementation = flag.String("topo_implementation", "zookeeper", "the topology implementation to use") @@ -192,6 +198,10 @@ var ( // factories has the factories for the Conn objects. factories = make(map[string]Factory) + + regions = cellsToRegionsMap{ + cellsToRegions: make(map[string]string), + } ) // RegisterFactory registers a Factory for an implementation for a Server. @@ -294,6 +304,32 @@ func (ts *Server) ConnForCell(ctx context.Context, cell string) (Conn, error) { return conn, nil } +// GetRegionByCell returns the region group this `cell` belongs to, if there's none, it returns the `cell` as region. +func GetRegionByCell(ctx context.Context, ts *Server, cell string) string { + regions.mu.Lock() + defer regions.mu.Unlock() + if region, ok := regions.cellsToRegions[cell]; ok { + return region + } + if ts != nil { + // lazily get the region from cell info if `regions.ts` is available + info, err := ts.GetCellInfo(ctx, cell, false) + if err == nil && info.Region != "" { + regions.cellsToRegions[cell] = info.Region + return info.Region + } + } + // for backward compatability + return cell +} + +// UpdateCellsToRegionsForTests overwrites the global map built by topo server init, and is meant for testing purpose only. +func UpdateCellsToRegionsForTests(cellsToRegions map[string]string) { + regions.mu.Lock() + defer regions.mu.Unlock() + regions.cellsToRegions = cellsToRegions +} + // Close will close all connections to underlying topo Server. // It will nil all member variables, so any further access will panic. func (ts *Server) Close() { diff --git a/go/vt/vtctl/cell_info.go b/go/vt/vtctl/cell_info.go index a8748f0b458..aa339c2c69a 100644 --- a/go/vt/vtctl/cell_info.go +++ b/go/vt/vtctl/cell_info.go @@ -39,13 +39,13 @@ func init() { addCommand(cellsGroupName, command{ "AddCellInfo", commandAddCellInfo, - "[-server_address ] [-root ] ", + "[-server_address ] [-root ] [-region ] ", "Registers a local topology service in a new cell by creating the CellInfo with the provided parameters. The address will be used to connect to the topology service, and we'll put Vitess data starting at the provided root."}) addCommand(cellsGroupName, command{ "UpdateCellInfo", commandUpdateCellInfo, - "[-server_address ] [-root ] ", + "[-server_address ] [-root ] [-region ] ", "Updates the content of a CellInfo with the provided parameters. If a value is empty, it is not updated. The CellInfo will be created if it doesn't exist."}) addCommand(cellsGroupName, command{ @@ -70,6 +70,7 @@ func init() { func commandAddCellInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { serverAddress := subFlags.String("server_address", "", "The address the topology server is using for that cell.") root := subFlags.String("root", "", "The root path the topology server is using for that cell.") + region := subFlags.String("region", "", "The region this cell belongs to.") if err := subFlags.Parse(args); err != nil { return err } @@ -81,12 +82,14 @@ func commandAddCellInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl return wr.TopoServer().CreateCellInfo(ctx, cell, &topodatapb.CellInfo{ ServerAddress: *serverAddress, Root: *root, + Region: *region, }) } func commandUpdateCellInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { serverAddress := subFlags.String("server_address", "", "The address the topology server is using for that cell.") root := subFlags.String("root", "", "The root path the topology server is using for that cell.") + region := subFlags.String("region", "", "The region this cell belongs to.") if err := subFlags.Parse(args); err != nil { return err } @@ -97,7 +100,8 @@ func commandUpdateCellInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags return wr.TopoServer().UpdateCellInfoFields(ctx, cell, func(ci *topodatapb.CellInfo) error { if (*serverAddress == "" || ci.ServerAddress == *serverAddress) && - (*root == "" || ci.Root == *root) { + (*root == "" || ci.Root == *root) && + (*region == "" || ci.Region == *region) { return topo.ErrNoUpdateNeeded } if *serverAddress != "" { @@ -106,6 +110,9 @@ func commandUpdateCellInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags if *root != "" { ci.Root = *root } + if *region != "" { + ci.Region = *region + } return nil }) } diff --git a/go/vt/vtgate/gateway/discoverygateway.go b/go/vt/vtgate/gateway/discoverygateway.go index 6962ad8bb02..34f28b32c92 100644 --- a/go/vt/vtgate/gateway/discoverygateway.go +++ b/go/vt/vtgate/gateway/discoverygateway.go @@ -86,7 +86,7 @@ type discoveryGateway struct { func createDiscoveryGateway(hc discovery.HealthCheck, topoServer *topo.Server, serv topo.SrvTopoServer, cell string, retryCount int) Gateway { dg := &discoveryGateway{ hc: hc, - tsc: discovery.NewTabletStatsCacheDoNotSetListener(cell), + tsc: discovery.NewTabletStatsCacheDoNotSetListener(topoServer, cell), topoServer: topoServer, srvTopoServer: serv, localCell: cell, @@ -228,7 +228,7 @@ func (dg *discoveryGateway) withRetry(ctx context.Context, target *querypb.Targe err = vterrors.New(vtrpcpb.Code_UNAVAILABLE, "no valid tablet") break } - shuffleTablets(tablets) + shuffleTablets(dg.localCell, tablets) // skip tablets we tried before var ts *discovery.TabletStats @@ -273,13 +273,51 @@ func (dg *discoveryGateway) withRetry(ctx context.Context, target *querypb.Targe return NewShardError(err, target, tabletLastUsed, inTransaction) } -func shuffleTablets(tablets []discovery.TabletStats) { - index := 0 +func shuffleTablets(cell string, tablets []discovery.TabletStats) { + sameCell, diffCell, sameCellMax := 0, 0, -1 length := len(tablets) - for i := length - 1; i > 0; i-- { - index = rand.Intn(i + 1) - tablets[i], tablets[index] = tablets[index], tablets[i] + + // move all same cell tablets to the front, this is O(n) + for { + sameCellMax = diffCell - 1 + sameCell = nextTablet(cell, tablets, sameCell, length, true) + diffCell = nextTablet(cell, tablets, diffCell, length, false) + // either no more diffs or no more same cells should stop the iteration + if sameCell < 0 || diffCell < 0 { + break + } + + if sameCell < diffCell { + // fast forward the `sameCell` lookup to `diffCell + 1`, `diffCell` unchanged + sameCell = diffCell + 1 + } else { + // sameCell > diffCell, swap needed + tablets[sameCell], tablets[diffCell] = tablets[diffCell], tablets[sameCell] + sameCell++ + diffCell++ + } + } + + //shuffle in same cell tablets + for i := sameCellMax; i > 0; i-- { + swap := rand.Intn(i + 1) + tablets[i], tablets[swap] = tablets[swap], tablets[i] + } + + //shuffle in diff cell tablets + for i, diffCellMin := length-1, sameCellMax+1; i > diffCellMin; i-- { + swap := rand.Intn(i-sameCellMax) + diffCellMin + tablets[i], tablets[swap] = tablets[swap], tablets[i] + } +} + +func nextTablet(cell string, tablets []discovery.TabletStats, offset, length int, sameCell bool) int { + for ; offset < length; offset++ { + if (tablets[offset].Tablet.Alias.Cell == cell) == sameCell { + return offset + } } + return -1 } func (dg *discoveryGateway) updateStats(target *querypb.Target, startTime time.Time, err error) { diff --git a/go/vt/vtgate/gateway/discoverygateway_test.go b/go/vt/vtgate/gateway/discoverygateway_test.go index 60368b71cda..8c3c5632976 100644 --- a/go/vt/vtgate/gateway/discoverygateway_test.go +++ b/go/vt/vtgate/gateway/discoverygateway_test.go @@ -127,6 +127,109 @@ func TestDiscoveryGatewayGetTablets(t *testing.T) { } } +func TestShuffleTablets(t *testing.T) { + defer topo.UpdateCellsToRegionsForTests(map[string]string{}) + topo.UpdateCellsToRegionsForTests(map[string]string{ + "cell1": "region1", + "cell2": "region1", + }) + + ts1 := discovery.TabletStats{ + Key: "t1", + Tablet: topo.NewTablet(10, "cell1", "host1"), + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Up: true, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + + ts2 := discovery.TabletStats{ + Key: "t2", + Tablet: topo.NewTablet(10, "cell1", "host2"), + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Up: true, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + + ts3 := discovery.TabletStats{ + Key: "t3", + Tablet: topo.NewTablet(10, "cell2", "host3"), + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Up: true, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + + ts4 := discovery.TabletStats{ + Key: "t4", + Tablet: topo.NewTablet(10, "cell2", "host4"), + Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA}, + Up: true, + Serving: true, + Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2}, + } + + sameCellTablets := []discovery.TabletStats{ts1, ts2} + diffCellTablets := []discovery.TabletStats{ts3, ts4} + mixedTablets := []discovery.TabletStats{ts1, ts2, ts3, ts4} + // repeat shuffling 10 times and everytime the same cell tablets should be in the front + for i := 0; i < 10; i++ { + shuffleTablets("cell1", sameCellTablets) + if (len(sameCellTablets) != 2) || + (sameCellTablets[0].Key != "t1" && sameCellTablets[0].Key != "t2") || + (sameCellTablets[1].Key != "t1" && sameCellTablets[1].Key != "t2") { + t.Errorf("should shuffle in only same cell tablets, got %+v", sameCellTablets) + } + + shuffleTablets("cell1", diffCellTablets) + if (len(diffCellTablets) != 2) || + (diffCellTablets[0].Key != "t3" && diffCellTablets[0].Key != "t4") || + (diffCellTablets[1].Key != "t3" && diffCellTablets[1].Key != "t4") { + t.Errorf("should shuffle in only diff cell tablets, got %+v", diffCellTablets) + } + + shuffleTablets("cell1", mixedTablets) + if len(mixedTablets) != 4 { + t.Errorf("should have 4 tablets, got %+v", mixedTablets) + } + + if (mixedTablets[0].Key != "t1" && mixedTablets[0].Key != "t2") || + (mixedTablets[1].Key != "t1" && mixedTablets[1].Key != "t2") { + t.Errorf("should have same cell tablets in the front, got %+v", mixedTablets) + } + + if (mixedTablets[2].Key != "t3" && mixedTablets[2].Key != "t4") || + (mixedTablets[3].Key != "t3" && mixedTablets[3].Key != "t4") { + t.Errorf("should have diff cell tablets in the rear, got %+v", mixedTablets) + } + } +} + +func TestDiscoveryGatewayGetTabletsWithRegion(t *testing.T) { + keyspace := "ks" + shard := "0" + hc := discovery.NewFakeHealthCheck() + dg := createDiscoveryGateway(hc, nil, nil, "local", 2).(*discoveryGateway) + topo.UpdateCellsToRegionsForTests(map[string]string{ + "local-west": "local", + "local-east": "local", + "local": "local", + "remote": "remote", + }) + + // replica should only use local ones + hc.Reset() + dg.tsc.ResetForTesting() + hc.AddTestTablet("remote", "1.1.1.1", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil) + ep1 := hc.AddTestTablet("local-west", "2.2.2.2", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil).Tablet() + ep2 := hc.AddTestTablet("local-east", "3.3.3.3", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil).Tablet() + tsl := dg.tsc.GetHealthyTabletStats(keyspace, shard, topodatapb.TabletType_REPLICA) + if len(tsl) != 2 || (!topo.TabletEquality(tsl[0].Tablet, ep1) && !topo.TabletEquality(tsl[0].Tablet, ep2)) { + t.Errorf("want %+v or %+v, got %+v", ep1, ep2, tsl) + } +} + func testDiscoveryGatewayGeneric(t *testing.T, streaming bool, f func(dg Gateway, target *querypb.Target) error) { keyspace := "ks" shard := "0" diff --git a/go/vt/vttablet/tabletmanager/binlog_players.go b/go/vt/vttablet/tabletmanager/binlog_players.go index 872dfb0d405..db0ead6e2cd 100644 --- a/go/vt/vttablet/tabletmanager/binlog_players.go +++ b/go/vt/vttablet/tabletmanager/binlog_players.go @@ -131,7 +131,7 @@ func newBinlogPlayerController(ts *topo.Server, vtClientFactory func() binlogpla // of whether the BinlogPlayerController is Start()'d or Stop()'d. // Use Close() after Stop() to finally close them and free their resources. healthCheck: healthCheck, - tabletStatsCache: discovery.NewTabletStatsCache(healthCheck, cell), + tabletStatsCache: discovery.NewTabletStatsCache(healthCheck, ts, cell), shardReplicationWatcher: discovery.NewShardReplicationWatcher(ts, healthCheck, cell, sourceShard.Keyspace, sourceShard.Shard, *healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency), } } diff --git a/go/vt/worker/legacy_split_clone.go b/go/vt/worker/legacy_split_clone.go index 745cac36d12..c29a26fad14 100644 --- a/go/vt/worker/legacy_split_clone.go +++ b/go/vt/worker/legacy_split_clone.go @@ -374,7 +374,7 @@ func (scw *LegacySplitCloneWorker) findTargets(ctx context.Context) error { // Initialize healthcheck and add destination shards to it. scw.healthCheck = discovery.NewHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) - scw.tsc = discovery.NewTabletStatsCache(scw.healthCheck, scw.cell) + scw.tsc = discovery.NewTabletStatsCache(scw.healthCheck, scw.wr.TopoServer(), scw.cell) for _, si := range scw.destinationShards { watcher := discovery.NewShardReplicationWatcher(scw.wr.TopoServer(), scw.healthCheck, scw.cell, si.Keyspace(), si.ShardName(), diff --git a/go/vt/worker/split_clone.go b/go/vt/worker/split_clone.go index 47987550615..250fc7d0307 100644 --- a/go/vt/worker/split_clone.go +++ b/go/vt/worker/split_clone.go @@ -555,7 +555,7 @@ func (scw *SplitCloneWorker) init(ctx context.Context) error { // Initialize healthcheck and add destination shards to it. scw.healthCheck = discovery.NewHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) - scw.tsc = discovery.NewTabletStatsCacheDoNotSetListener(scw.cell) + scw.tsc = discovery.NewTabletStatsCacheDoNotSetListener(scw.wr.TopoServer(), scw.cell) // We set sendDownEvents=true because it's required by TabletStatsCache. scw.healthCheck.SetListener(scw, true /* sendDownEvents */) diff --git a/go/vt/worker/topo_utils.go b/go/vt/worker/topo_utils.go index edb3e9eafcb..2b280a042b0 100644 --- a/go/vt/worker/topo_utils.go +++ b/go/vt/worker/topo_utils.go @@ -50,7 +50,7 @@ func FindHealthyRdonlyTablet(ctx context.Context, wr *wrangler.Wrangler, tsc *di if tsc == nil { // No healthcheck instance provided. Create one. healthCheck := discovery.NewHealthCheck(*healthcheckRetryDelay, *healthCheckTimeout) - tsc = discovery.NewTabletStatsCache(healthCheck, cell) + tsc = discovery.NewTabletStatsCache(healthCheck, wr.TopoServer(), cell) watcher := discovery.NewShardReplicationWatcher(wr.TopoServer(), healthCheck, cell, keyspace, shard, *healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) defer watcher.Stop() defer healthCheck.Close() diff --git a/go/vt/wrangler/keyspace.go b/go/vt/wrangler/keyspace.go index 699a9dea0a7..3900b4917c9 100644 --- a/go/vt/wrangler/keyspace.go +++ b/go/vt/wrangler/keyspace.go @@ -507,7 +507,7 @@ func (wr *Wrangler) waitForDrainInCell(ctx context.Context, cell, keyspace, shar // Create the healthheck module, with a cache. hc := discovery.NewHealthCheck(healthcheckRetryDelay, healthCheckTimeout) defer hc.Close() - tsc := discovery.NewTabletStatsCache(hc, cell) + tsc := discovery.NewTabletStatsCache(hc, wr.TopoServer(), cell) // Create a tablet watcher. watcher := discovery.NewShardReplicationWatcher(wr.TopoServer(), hc, cell, keyspace, shard, healthCheckTopologyRefresh, discovery.DefaultTopoReadConcurrency) diff --git a/proto/topodata.proto b/proto/topodata.proto index 55805771d71..d5beadc8825 100644 --- a/proto/topodata.proto +++ b/proto/topodata.proto @@ -323,4 +323,8 @@ message CellInfo { // Root is the path to store data in. It is only used when talking // to server_address. string root = 2; + + // Region is a group this cell belongs to. Used by vtgate to route traffic to + // other cells (in same region) when there is no available tablet in the current cell. + string region = 3; } diff --git a/py/vtproto/topodata_pb2.py b/py/vtproto/topodata_pb2.py index 729e0abbbc4..fb3e75687c8 100644 --- a/py/vtproto/topodata_pb2.py +++ b/py/vtproto/topodata_pb2.py @@ -20,9 +20,8 @@ name='topodata.proto', package='topodata', syntax='proto3', - serialized_pb=_b('\n\x0etopodata.proto\x12\x08topodata\"&\n\x08KeyRange\x12\r\n\x05start\x18\x01 \x01(\x0c\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x0c\"(\n\x0bTabletAlias\x12\x0c\n\x04\x63\x65ll\x18\x01 \x01(\t\x12\x0b\n\x03uid\x18\x02 \x01(\r\"\xb6\x03\n\x06Tablet\x12$\n\x05\x61lias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12\x10\n\x08hostname\x18\x02 \x01(\t\x12/\n\x08port_map\x18\x04 \x03(\x0b\x32\x1d.topodata.Tablet.PortMapEntry\x12\x10\n\x08keyspace\x18\x05 \x01(\t\x12\r\n\x05shard\x18\x06 \x01(\t\x12%\n\tkey_range\x18\x07 \x01(\x0b\x32\x12.topodata.KeyRange\x12\"\n\x04type\x18\x08 \x01(\x0e\x32\x14.topodata.TabletType\x12\x18\n\x10\x64\x62_name_override\x18\t \x01(\t\x12(\n\x04tags\x18\n \x03(\x0b\x32\x1a.topodata.Tablet.TagsEntry\x12\x16\n\x0emysql_hostname\x18\x0c \x01(\t\x12\x12\n\nmysql_port\x18\r \x01(\x05\x1a.\n\x0cPortMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04J\x04\x08\x0b\x10\x0c\"\xcb\x04\n\x05Shard\x12+\n\x0cmaster_alias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x30\n\x0cserved_types\x18\x03 \x03(\x0b\x32\x1a.topodata.Shard.ServedType\x12\x32\n\rsource_shards\x18\x04 \x03(\x0b\x32\x1b.topodata.Shard.SourceShard\x12\r\n\x05\x63\x65lls\x18\x05 \x03(\t\x12\x36\n\x0ftablet_controls\x18\x06 \x03(\x0b\x32\x1d.topodata.Shard.TabletControl\x1a\x46\n\nServedType\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x1ar\n\x0bSourceShard\x12\x0b\n\x03uid\x18\x01 \x01(\r\x12\x10\n\x08keyspace\x18\x02 \x01(\t\x12\r\n\x05shard\x18\x03 \x01(\t\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x1a\x84\x01\n\rTabletControl\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x12\x1d\n\x15\x64isable_query_service\x18\x03 \x01(\x08\x12\x1a\n\x12\x62lacklisted_tables\x18\x04 \x03(\t\"\xf5\x01\n\x08Keyspace\x12\x1c\n\x14sharding_column_name\x18\x01 \x01(\t\x12\x36\n\x14sharding_column_type\x18\x02 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12\x33\n\x0cserved_froms\x18\x04 \x03(\x0b\x32\x1d.topodata.Keyspace.ServedFrom\x1aX\n\nServedFrom\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x12\x10\n\x08keyspace\x18\x03 \x01(\tJ\x04\x08\x03\x10\x04\"w\n\x10ShardReplication\x12.\n\x05nodes\x18\x01 \x03(\x0b\x32\x1f.topodata.ShardReplication.Node\x1a\x33\n\x04Node\x12+\n\x0ctablet_alias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\"E\n\x0eShardReference\x12\x0c\n\x04name\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\"\x9c\x03\n\x0bSrvKeyspace\x12;\n\npartitions\x18\x01 \x03(\x0b\x32\'.topodata.SrvKeyspace.KeyspacePartition\x12\x1c\n\x14sharding_column_name\x18\x02 \x01(\t\x12\x36\n\x14sharding_column_type\x18\x03 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12\x35\n\x0bserved_from\x18\x04 \x03(\x0b\x32 .topodata.SrvKeyspace.ServedFrom\x1ar\n\x11KeyspacePartition\x12)\n\x0bserved_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\x32\n\x10shard_references\x18\x02 \x03(\x0b\x32\x18.topodata.ShardReference\x1aI\n\nServedFrom\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\x10\n\x08keyspace\x18\x02 \x01(\tJ\x04\x08\x05\x10\x06\"0\n\x08\x43\x65llInfo\x12\x16\n\x0eserver_address\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x02 \x01(\t*2\n\x0eKeyspaceIdType\x12\t\n\x05UNSET\x10\x00\x12\n\n\x06UINT64\x10\x01\x12\t\n\x05\x42YTES\x10\x02*\x90\x01\n\nTabletType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06MASTER\x10\x01\x12\x0b\n\x07REPLICA\x10\x02\x12\n\n\x06RDONLY\x10\x03\x12\t\n\x05\x42\x41TCH\x10\x03\x12\t\n\x05SPARE\x10\x04\x12\x10\n\x0c\x45XPERIMENTAL\x10\x05\x12\n\n\x06\x42\x41\x43KUP\x10\x06\x12\x0b\n\x07RESTORE\x10\x07\x12\x0b\n\x07\x44RAINED\x10\x08\x1a\x02\x10\x01\x42\x11\n\x0fio.vitess.protob\x06proto3') + serialized_pb=_b('\n\x0etopodata.proto\x12\x08topodata\"&\n\x08KeyRange\x12\r\n\x05start\x18\x01 \x01(\x0c\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x0c\"(\n\x0bTabletAlias\x12\x0c\n\x04\x63\x65ll\x18\x01 \x01(\t\x12\x0b\n\x03uid\x18\x02 \x01(\r\"\xb6\x03\n\x06Tablet\x12$\n\x05\x61lias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12\x10\n\x08hostname\x18\x02 \x01(\t\x12/\n\x08port_map\x18\x04 \x03(\x0b\x32\x1d.topodata.Tablet.PortMapEntry\x12\x10\n\x08keyspace\x18\x05 \x01(\t\x12\r\n\x05shard\x18\x06 \x01(\t\x12%\n\tkey_range\x18\x07 \x01(\x0b\x32\x12.topodata.KeyRange\x12\"\n\x04type\x18\x08 \x01(\x0e\x32\x14.topodata.TabletType\x12\x18\n\x10\x64\x62_name_override\x18\t \x01(\t\x12(\n\x04tags\x18\n \x03(\x0b\x32\x1a.topodata.Tablet.TagsEntry\x12\x16\n\x0emysql_hostname\x18\x0c \x01(\t\x12\x12\n\nmysql_port\x18\r \x01(\x05\x1a.\n\x0cPortMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04J\x04\x08\x0b\x10\x0c\"\xcb\x04\n\x05Shard\x12+\n\x0cmaster_alias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x30\n\x0cserved_types\x18\x03 \x03(\x0b\x32\x1a.topodata.Shard.ServedType\x12\x32\n\rsource_shards\x18\x04 \x03(\x0b\x32\x1b.topodata.Shard.SourceShard\x12\r\n\x05\x63\x65lls\x18\x05 \x03(\t\x12\x36\n\x0ftablet_controls\x18\x06 \x03(\x0b\x32\x1d.topodata.Shard.TabletControl\x1a\x46\n\nServedType\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x1ar\n\x0bSourceShard\x12\x0b\n\x03uid\x18\x01 \x01(\r\x12\x10\n\x08keyspace\x18\x02 \x01(\t\x12\r\n\x05shard\x18\x03 \x01(\t\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x1a\x84\x01\n\rTabletControl\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x12\x1d\n\x15\x64isable_query_service\x18\x03 \x01(\x08\x12\x1a\n\x12\x62lacklisted_tables\x18\x04 \x03(\t\"\xf5\x01\n\x08Keyspace\x12\x1c\n\x14sharding_column_name\x18\x01 \x01(\t\x12\x36\n\x14sharding_column_type\x18\x02 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12\x33\n\x0cserved_froms\x18\x04 \x03(\x0b\x32\x1d.topodata.Keyspace.ServedFrom\x1aX\n\nServedFrom\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x12\x10\n\x08keyspace\x18\x03 \x01(\tJ\x04\x08\x03\x10\x04\"w\n\x10ShardReplication\x12.\n\x05nodes\x18\x01 \x03(\x0b\x32\x1f.topodata.ShardReplication.Node\x1a\x33\n\x04Node\x12+\n\x0ctablet_alias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\"E\n\x0eShardReference\x12\x0c\n\x04name\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\"\x9c\x03\n\x0bSrvKeyspace\x12;\n\npartitions\x18\x01 \x03(\x0b\x32\'.topodata.SrvKeyspace.KeyspacePartition\x12\x1c\n\x14sharding_column_name\x18\x02 \x01(\t\x12\x36\n\x14sharding_column_type\x18\x03 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12\x35\n\x0bserved_from\x18\x04 \x03(\x0b\x32 .topodata.SrvKeyspace.ServedFrom\x1ar\n\x11KeyspacePartition\x12)\n\x0bserved_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\x32\n\x10shard_references\x18\x02 \x03(\x0b\x32\x18.topodata.ShardReference\x1aI\n\nServedFrom\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\x10\n\x08keyspace\x18\x02 \x01(\tJ\x04\x08\x05\x10\x06\"@\n\x08\x43\x65llInfo\x12\x16\n\x0eserver_address\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x02 \x01(\t\x12\x0e\n\x06region\x18\x03 \x01(\t*2\n\x0eKeyspaceIdType\x12\t\n\x05UNSET\x10\x00\x12\n\n\x06UINT64\x10\x01\x12\t\n\x05\x42YTES\x10\x02*\x90\x01\n\nTabletType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06MASTER\x10\x01\x12\x0b\n\x07REPLICA\x10\x02\x12\n\n\x06RDONLY\x10\x03\x12\t\n\x05\x42\x41TCH\x10\x03\x12\t\n\x05SPARE\x10\x04\x12\x10\n\x0c\x45XPERIMENTAL\x10\x05\x12\n\n\x06\x42\x41\x43KUP\x10\x06\x12\x0b\n\x07RESTORE\x10\x07\x12\x0b\n\x07\x44RAINED\x10\x08\x1a\x02\x10\x01\x42\x11\n\x0fio.vitess.protob\x06proto3') ) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) _KEYSPACEIDTYPE = _descriptor.EnumDescriptor( name='KeyspaceIdType', @@ -45,8 +44,8 @@ ], containing_type=None, options=None, - serialized_start=2046, - serialized_end=2096, + serialized_start=2062, + serialized_end=2112, ) _sym_db.RegisterEnumDescriptor(_KEYSPACEIDTYPE) @@ -100,8 +99,8 @@ ], containing_type=None, options=_descriptor._ParseOptions(descriptor_pb2.EnumOptions(), _b('\020\001')), - serialized_start=2099, - serialized_end=2243, + serialized_start=2115, + serialized_end=2259, ) _sym_db.RegisterEnumDescriptor(_TABLETTYPE) @@ -920,6 +919,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + _descriptor.FieldDescriptor( + name='region', full_name='topodata.CellInfo.region', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -933,7 +939,7 @@ oneofs=[ ], serialized_start=1996, - serialized_end=2044, + serialized_end=2060, ) _TABLET_PORTMAPENTRY.containing_type = _TABLET @@ -981,6 +987,7 @@ DESCRIPTOR.message_types_by_name['CellInfo'] = _CELLINFO DESCRIPTOR.enum_types_by_name['KeyspaceIdType'] = _KEYSPACEIDTYPE DESCRIPTOR.enum_types_by_name['TabletType'] = _TABLETTYPE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) KeyRange = _reflection.GeneratedProtocolMessageType('KeyRange', (_message.Message,), dict( DESCRIPTOR = _KEYRANGE, @@ -1126,9 +1133,4 @@ _TABLET_PORTMAPENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) _TABLET_TAGSENTRY.has_options = True _TABLET_TAGSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) -import grpc -from grpc.beta import implementations as beta_implementations -from grpc.beta import interfaces as beta_interfaces -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope)