Add realtime health stats to vtctld /api/keyspace/ks/tablets API

go/vt/vtctld/api.go

@@ -42,6 +42,7 @@ import (
 
 	"vitess.io/vitess/go/vt/mysqlctl"
 	logutilpb "vitess.io/vitess/go/vt/proto/logutil"
+	querypb "vitess.io/vitess/go/vt/proto/query"
 	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/proto/vttime"
 )
@@ -60,8 +61,16 @@ const (
 	jsonContentType = "application/json; charset=utf-8"
 )
 
-// TabletWithURL wraps topo.Tablet and adds a URL property.
-type TabletWithURL struct {
+// TabletStats represents realtime stats from a discovery.LegacyTabletStats struct.
+type TabletStats struct {
+	LastError     string                 `json:"last_error,omitempty"`
+	RealtimeStats *querypb.RealtimeStats `json:"realtime,omitempty"`
+	Serving       bool                   `json:"serving,omitempty"`
+	Up            bool                   `json:"up,omitempty"`
+}
+
+// TabletWithStatsAndURL wraps topo.Tablet, adding a URL property and optional realtime stats.
+type TabletWithStatsAndURL struct {
 	Alias               *topodatapb.TabletAlias `json:"alias,omitempty"`
 	Hostname            string                  `json:"hostname,omitempty"`
 	PortMap             map[string]int32        `json:"port_map,omitempty"`
@@ -74,9 +83,48 @@ type TabletWithURL struct {
 	MysqlHostname       string                  `json:"mysql_hostname,omitempty"`
 	MysqlPort           int32                   `json:"mysql_port,omitempty"`
 	MasterTermStartTime *vttime.Time            `json:"master_term_start_time,omitempty"`
+	Stats               *TabletStats            `json:"stats,omitempty"`
 	URL                 string                  `json:"url,omitempty"`
 }
 
+func NewTabletWithStatsAndURL(t *topodatapb.Tablet, realtimeStats *realtimeStats) (*TabletWithStatsAndURL, error) {
+	tablet := &TabletWithStatsAndURL{
+		Alias:               t.Alias,
+		Hostname:            t.Hostname,
+		PortMap:             t.PortMap,
+		Keyspace:            t.Keyspace,
+		Shard:               t.Shard,
+		KeyRange:            t.KeyRange,
+		Type:                t.Type,
+		DbNameOverride:      t.DbNameOverride,
+		Tags:                t.Tags,
+		MysqlHostname:       t.MysqlHostname,
+		MysqlPort:           t.MysqlPort,
+		MasterTermStartTime: t.MasterTermStartTime,
+	}
+
+	if *proxyTablets {
+		tablet.URL = fmt.Sprintf("/vttablet/%s-%d/debug/status", t.Alias.Cell, t.Alias.Uid)
+	} else {
+		tablet.URL = "http://" + netutil.JoinHostPort(t.Hostname, t.PortMap["vt"])
+	}
+
+	if realtimeStats != nil {
+		if stats, err := realtimeStats.tabletStats(tablet.Alias); err == nil {
+			tablet.Stats = &TabletStats{
+				RealtimeStats: stats.Stats,
+				Serving:       stats.Serving,
+				Up:            stats.Up,
+			}
+			if stats.LastError != nil {
+				tablet.Stats.LastError = stats.LastError.Error()
+			}
+		}
+	}
+
+	return tablet, nil
+}
+
 func httpErrorf(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) {
 	errMsg := fmt.Sprintf(format, args...)
 	log.Errorf("HTTP error on %v: %v, request: %#v", r.URL.Path, errMsg, r)
@@ -218,10 +266,23 @@ func initAPI(ctx context.Context, ts *topo.Server, actions *ActionRepository, re
 				return nil, err
 			}
 		}
-		tablets := [](*topodatapb.Tablet){}
+
+		if err := r.ParseForm(); err != nil {
+			return nil, err
+		}
+		cell := r.FormValue("cell")
+		cells := r.FormValue("cells")
+		filterCells := []string{} // empty == all cells
+		if cell != "" {
+			filterCells = []string{cell} // single cell
+		} else if cells != "" {
+			filterCells = strings.Split(cells, ",") // list of cells
+		}
+
+		tablets := [](*TabletWithStatsAndURL){}
 		for _, shard := range shardNames {
 			// Get tablets for this shard.
-			tabletAliases, err := ts.FindAllTabletAliasesInShard(ctx, keyspace, shard)
+			tabletAliases, err := ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, filterCells)
 			if err != nil && !topo.IsErrType(err, topo.PartialResult) {
 				return nil, err
 			}
@@ -230,7 +291,11 @@ func initAPI(ctx context.Context, ts *topo.Server, actions *ActionRepository, re
 				if err != nil {
 					return nil, err
 				}
-				tablets = append(tablets, t.Tablet)
+				tablet, err := NewTabletWithStatsAndURL(t.Tablet, realtimeStats)
+				if err != nil {
+					return nil, err
+				}
+				tablets = append(tablets, tablet)
 			}
 		}
 		return tablets, nil
@@ -402,26 +467,7 @@ func initAPI(ctx context.Context, ts *topo.Server, actions *ActionRepository, re
 			return nil, err
 		}
 
-		tab := &TabletWithURL{
-			Alias:               t.Alias,
-			Hostname:            t.Hostname,
-			PortMap:             t.PortMap,
-			Keyspace:            t.Keyspace,
-			Shard:               t.Shard,
-			KeyRange:            t.KeyRange,
-			Type:                t.Type,
-			DbNameOverride:      t.DbNameOverride,
-			Tags:                t.Tags,
-			MysqlHostname:       t.MysqlHostname,
-			MysqlPort:           t.MysqlPort,
-			MasterTermStartTime: t.MasterTermStartTime,
-		}
-		if *proxyTablets {
-			tab.URL = fmt.Sprintf("/vttablet/%s-%d/debug/status", t.Alias.Cell, t.Alias.Uid)
-		} else {
-			tab.URL = "http://" + netutil.JoinHostPort(t.Hostname, t.PortMap["vt"])
-		}
-		return tab, nil
+		return NewTabletWithStatsAndURL(t.Tablet, nil)
 	})
 
 	// Healthcheck real time status per (cell, keyspace, tablet type, metric).

go/vt/vtctld/api_test.go

@@ -126,6 +126,49 @@ func TestAPI(t *testing.T) {
 	realtimeStats.StatsUpdate(ts5)
 	realtimeStats.StatsUpdate(ts6)
 
+	// all-tablets response for keyspace/ks1/tablets/ endpoints
+	keyspaceKs1AllTablets := `[
+		{
+			"alias": {
+				"cell": "cell1",
+				"uid": 100
+			},
+			"port_map": {
+				"vt": 100
+			},
+			"keyspace": "ks1",
+			"shard": "-80",
+			"key_range": {
+				"end": "gA=="
+			},
+			"type": 2,
+			"stats": {
+				"realtime": {
+					"seconds_behind_master": 100
+				},
+				"serving": true,
+				"up": true
+			},
+			"url": "http://:100"
+		},
+		{
+			"alias": {
+				"cell": "cell2",
+				"uid": 200
+			},
+			"port_map": {
+				"vt": 200
+			},
+			"keyspace": "ks1",
+			"shard": "-80",
+			"key_range": {
+				"end": "gA=="
+			},
+			"type": 2,
+			"url": "http://:200"
+		}
+	]`
+
 	// Test cases.
 	table := []struct {
 		method, path, body, want string
@@ -139,6 +182,58 @@ func TestAPI(t *testing.T) {
 		// Cells
 		{"GET", "cells", "", `["cell1","cell2"]`},
 
+		// Keyspace
+		{"GET", "keyspace/doesnt-exist/tablets/", "", ``},
+		{"GET", "keyspace/ks1/tablets/", "", keyspaceKs1AllTablets},
+		{"GET", "keyspace/ks1/tablets/-80", "", keyspaceKs1AllTablets},
+		{"GET", "keyspace/ks1/tablets/80-", "", `[]`},
+		{"GET", "keyspace/ks1/tablets/?cells=cell1,cell2", "", keyspaceKs1AllTablets},
+		{"GET", "keyspace/ks1/tablets/?cells=cell1", "", `[
+			{
+				"alias": {
+					"cell": "cell1",
+					"uid": 100
+				},
+				"port_map": {
+					"vt": 100
+				},
+				"keyspace": "ks1",
+				"shard": "-80",
+				"key_range": {
+					"end": "gA=="
+				},
+				"type": 2,
+				"stats": {
+					"realtime": {
+						"seconds_behind_master": 100
+					},
+					"serving": true,
+					"up": true
+				},
+				"url": "http://:100"
+			}
+		]`},
+		{"GET", "keyspace/ks1/tablets/?cells=cell3", "", `[]`},
+		{"GET", "keyspace/ks1/tablets/?cell=cell2", "", `[
+			{
+				"alias": {
+					"cell": "cell2",
+					"uid": 200
+				},
+				"port_map": {
+					"vt": 200
+				},
+				"keyspace": "ks1",
+				"shard": "-80",
+				"key_range": {
+					"end": "gA=="
+				},
+				"type": 2,
+				"url": "http://:200"
+			}
+		]`},
+		{"GET", "keyspace/ks1/tablets/?cell=cell3", "", `[]`},
+
 		// Keyspaces
 		{"GET", "keyspaces", "", `["ks1", "ks3"]`},
 		{"GET", "keyspaces/ks1", "", `{