etcd all about when hung up, the user can use vtgate normal

go/vt/discovery/tablet_stats_cache_wait_test.go

@@ -84,10 +84,17 @@ func TestFindAllKeyspaceShards(t *testing.T) {
 	}
 
 	// Get it.
-	ks, err = FindAllTargets(ctx, rs, "cell1", []topodatapb.TabletType{topodatapb.TabletType_MASTER})
-	if err != nil {
-		t.Errorf("unexpected error: %v", err)
+	for {
+		ks, err = FindAllTargets(ctx, rs, "cell1", []topodatapb.TabletType{topodatapb.TabletType_MASTER})
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if ks != nil {
+			break
+		}
+		time.Sleep(time.Millisecond)
 	}
+
 	if !reflect.DeepEqual(ks, []*querypb.Target{
 		{
 			Cell:       "cell1",

go/vt/srvtopo/resilient_server.go

@@ -207,53 +207,51 @@ func (server *ResilientServer) GetSrvKeyspaceNames(ctx context.Context, cell str
 	}
 	server.mutex.Unlock()
 
-	// Lock the entry, and do everything holding the lock.  This
-	// means two concurrent requests will only issue one
-	// underlying query.
-	entry.mutex.Lock()
-	defer entry.mutex.Unlock()
-
 	// If it is not time to check again, then return either the cached
 	// value or the cached error
 	cacheValid := entry.value != nil && time.Since(entry.insertionTime) < server.cacheTTL
 	shouldRefresh := time.Since(entry.lastQueryTime) > server.cacheRefresh
 
-	if !shouldRefresh {
-		if cacheValid {
-			return entry.value, nil
-		}
-		return nil, entry.lastError
+	if !shouldRefresh && cacheValid {
+		return entry.value, nil
 	}
 
-	// Not in cache or needs refresh so try to get the real value.
-	// We use the context that issued the query here.
-	result, err := server.topoServer.GetSrvKeyspaceNames(ctx, cell)
-	if err == nil {
-		// save the value we got and the current time in the cache
-		entry.insertionTime = time.Now()
-		entry.value = result
-	} else {
-		if entry.insertionTime.IsZero() {
-			server.counts.Add(errorCategory, 1)
-			log.Errorf("GetSrvKeyspaceNames(%v, %v) failed: %v (no cached value, caching and returning error)", ctx, cell, err)
-
-		} else if cacheValid {
-			server.counts.Add(cachedCategory, 1)
-			log.Warningf("GetSrvKeyspaceNames(%v, %v) failed: %v (returning cached value: %v %v)", ctx, cell, err, entry.value, entry.lastError)
-			result = entry.value
-			err = nil
+	go func() {
+		// Lock the entry, and do everything holding the lock.  This
+		// means two concurrent requests will only issue one
+		// underlying query.
+		entry.mutex.Lock()
+		defer entry.mutex.Unlock()
+
+		if entry.value != nil && time.Since(entry.insertionTime) < server.cacheTTL {
+			return
+		}
+
+		// Not in cache or needs refresh so try to get the real value.
+		// We use the context that issued the query here.
+		result, err := server.topoServer.GetSrvKeyspaceNames(ctx, cell)
+		if err == nil {
+			// save the value we got and the current time in the cache
+			entry.insertionTime = time.Now()
+			entry.value = result
 		} else {
-			server.counts.Add(errorCategory, 1)
-			log.Errorf("GetSrvKeyspaceNames(%v, %v) failed: %v (cached value expired)", ctx, cell, err)
-			entry.insertionTime = time.Time{}
-			entry.value = nil
+			if entry.insertionTime.IsZero() {
+				server.counts.Add(errorCategory, 1)
+				log.Errorf("GetSrvKeyspaceNames(%v, %v) failed: %v (no cached value, caching and returning error)", ctx, cell, err)
+			} else if cacheValid {
+				server.counts.Add(cachedCategory, 1)
+				log.Warningf("GetSrvKeyspaceNames(%v, %v) failed: %v (returning cached value: %v %v)", ctx, cell, err, entry.value, entry.lastError)
+			} else {
+				server.counts.Add(errorCategory, 1)
+				log.Errorf("GetSrvKeyspaceNames(%v, %v) failed: %v (cached value expired, but also returning cached value: %v %v)", ctx, cell, err, entry.value, entry.lastError)
+			}
 		}
-	}
+		entry.lastError = err
+		entry.lastQueryTime = time.Now()
+		entry.lastErrorCtx = ctx
+	}()
 
-	entry.lastError = err
-	entry.lastQueryTime = time.Now()
-	entry.lastErrorCtx = ctx
-	return result, err
+	return entry.value, entry.lastError
 }
 
 func (server *ResilientServer) getSrvKeyspaceEntry(cell, keyspace string) *srvKeyspaceEntry {

go/vt/srvtopo/resilient_server_flaky_test.go

@@ -394,10 +394,20 @@ func TestGetSrvKeyspaceNames(t *testing.T) {
 	ts.UpdateSrvKeyspace(context.Background(), "test_cell", "test_ks2", want)
 
 	ctx := context.Background()
-	names, err := rs.GetSrvKeyspaceNames(ctx, "test_cell")
-	if err != nil {
-		t.Errorf("GetSrvKeyspaceNames unexpected error %v", err)
+
+	var names []string
+	var err error
+	for {
+		names, err = rs.GetSrvKeyspaceNames(ctx, "test_cell")
+		if err != nil {
+			t.Errorf("GetSrvKeyspaceNames unexpected error %v", err)
+		}
+		if names != nil {
+			break
+		}
+		time.Sleep(time.Millisecond)
 	}
+
 	wantNames := []string{"test_ks", "test_ks2"}
 
 	if !reflect.DeepEqual(names, wantNames) {

go/vt/vtgate/vindexes/lookup_internal.go

@@ -137,7 +137,7 @@ func (lkp *lookupInternal) Create(vcursor VCursor, rowsColValues [][]sqltypes.Va
 //
 // Given the following information in a vindex table with two columns:
 //
-//      +------------------+-----------+--------+
+//  +------------------+-----------+--------+
 //	| hex(keyspace_id) | a         | b      |
 //	+------------------+-----------+--------+
 //	| 52CB7B1B31B2222E | valuea    | valueb |

go/vt/vtgate/vindexes/vschema.go

@@ -51,7 +51,7 @@ type Table struct {
 	Pinned         []byte               `json:"pinned,omitempty"`
 }
 
-// Keyspace contains the keyspcae info for each Table.
+// Keyspace contains the keyspace info for each Table.
 type Keyspace struct {
 	Name    string
 	Sharded bool
@@ -404,7 +404,7 @@ func (vschema *VSchema) FindTableOrVindex(keyspace, name string) (*Table, Vindex
 }
 
 // FindVindex finds a vindex by name. If a keyspace is specified, only vindexes
-// from that keyspace are searched. If no kesypace is specified, then a vindex
+// from that keyspace are searched. If no keyspace is specified, then a vindex
 // is returned only if its name is unique across all keyspaces. The function
 // returns an error only if the vindex name is ambiguous.
 func (vschema *VSchema) FindVindex(keyspace, name string) (Vindex, error) {