From ba4a04ba30e3db5bd0a4c079a3eb2bcdaa387cb6 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Fri, 5 Jun 2020 14:37:49 -0700 Subject: [PATCH] Rebuild SrvVSchema during InitTablet. We used to rebuild SrvVSchema in all cells as part of running GetOrCreateShard from any cell. That caused VSchema problems to spread instantly to all cells at once, so we changed EnsureVSchema to only rebuild SrvVSchema in the local cell. We believed that at least one tablet in a given cell would still end up rebuilding SrvVschema in the cell since they all call GetOrCreateShard at startup. However, I didn't realize that GetOrCreateShard short-circuits if the shard already exists, so it was skipping EnsureVSchema entirely. This moves the rebuild of SrvVSchema to be an explicit step in InitTablet alongside the rebuild of the keyspace graph. This keeps GetOrCreateShard as a task that only needs to be done by one tablet globally because it affects global topo. The rebuilding of per-cell records should be done by a tablet in that cell. Signed-off-by: Anthony Yeh --- go/vt/topo/shard.go | 2 +- go/vt/topo/vschema.go | 8 +------ go/vt/vtctl/vtctl.go | 18 ++++++---------- go/vt/vttablet/tabletmanager/init_tablet.go | 24 +++++++++++++++++++-- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 44f165b1e52..c3f71632122 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -342,7 +342,7 @@ func (ts *Server) GetOrCreateShard(ctx context.Context, keyspace, shard, cell st // make sure a valid vschema has been loaded log.Info("EnsureVSchema %s/%s", keyspace, shard) - if err = ts.EnsureVSchema(ctx, keyspace, []string{cell}); err != nil { + if err = ts.EnsureVSchema(ctx, keyspace); err != nil { return nil, vterrors.Wrapf(err, "EnsureVSchema(%v) failed", keyspace) } diff --git a/go/vt/topo/vschema.go b/go/vt/topo/vschema.go index 40837595c8d..34a6739e9ba 100644 --- a/go/vt/topo/vschema.go +++ b/go/vt/topo/vschema.go @@ -72,7 +72,7 @@ func (ts *Server) GetVSchema(ctx context.Context, keyspace string) (*vschemapb.K } // EnsureVSchema makes sure that a vschema is present for this keyspace or creates a blank one if it is missing -func (ts *Server) EnsureVSchema(ctx context.Context, keyspace string, cells []string) error { +func (ts *Server) EnsureVSchema(ctx context.Context, keyspace string) error { vschema, err := ts.GetVSchema(ctx, keyspace) if err != nil && !IsErrType(err, NoNode) { log.Info("error in getting vschema for keyspace %s: %v", keyspace, err) @@ -88,12 +88,6 @@ func (ts *Server) EnsureVSchema(ctx context.Context, keyspace string, cells []st return err } } - - err = ts.RebuildSrvVSchema(ctx, cells) - if err != nil { - log.Errorf("could not rebuild SrvVschema after creating keyspace: %v", err) - return err - } return nil } diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index cf19f6707cb..2acf9a970f9 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -1666,18 +1666,14 @@ func commandCreateKeyspace(ctx context.Context, wr *wrangler.Wrangler, subFlags wr.Logger().Infof("keyspace %v already exists (ignoring error with -force)", keyspace) err = nil } + if err != nil { + return err + } if !*allowEmptyVSchema { - cells, err := wr.TopoServer().GetKnownCells(ctx) - if err != nil { - return fmt.Errorf("GetKnownCells failed: %v", err) + if err := wr.TopoServer().EnsureVSchema(ctx, keyspace); err != nil { + return err } - - err = wr.TopoServer().EnsureVSchema(ctx, keyspace, cells) - } - - if err != nil { - return err } if ktype == topodatapb.KeyspaceType_SNAPSHOT { @@ -1703,9 +1699,9 @@ func commandCreateKeyspace(ctx context.Context, wr *wrangler.Wrangler, subFlags wr.Logger().Infof("error from SaveVSchema %v:%v", vs, err) return err } - return wr.TopoServer().RebuildSrvVSchema(ctx, []string{} /* cells */) } - return nil + + return wr.TopoServer().RebuildSrvVSchema(ctx, []string{} /* cells */) } func commandDeleteKeyspace(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { diff --git a/go/vt/vttablet/tabletmanager/init_tablet.go b/go/vt/vttablet/tabletmanager/init_tablet.go index cb029fa854f..ea4edc985a6 100644 --- a/go/vt/vttablet/tabletmanager/init_tablet.go +++ b/go/vt/vttablet/tabletmanager/init_tablet.go @@ -155,10 +155,30 @@ func (agent *ActionAgent) InitTablet(port, gRPCPort int32) error { case err == nil: // NOOP case topo.IsErrType(err, topo.NoNode): - // try to RebuildKeyspace here but ignore errors if it fails + // Try to RebuildKeyspace here but ignore errors if it fails. + // It will fail until at least one tablet is up for every shard. topotools.RebuildKeyspace(ctx, logutil.NewConsoleLogger(), agent.TopoServer, *initKeyspace, []string{agent.TabletAlias.Cell}) default: - return vterrors.Wrap(err, "InitTablet failed to read srvKeyspace") + return vterrors.Wrap(err, "InitTablet failed to read SrvKeyspace") + } + + // Rebuild vschema graph if this is the first tablet in this keyspace/cell. + srvVSchema, err := agent.TopoServer.GetSrvVSchema(ctx, agent.TabletAlias.Cell) + switch { + case err == nil: + // Check if vschema was rebuilt after the initial creation of the keyspace. + if _, keyspaceExists := srvVSchema.GetKeyspaces()[*initKeyspace]; !keyspaceExists { + if err := agent.TopoServer.RebuildSrvVSchema(ctx, []string{agent.TabletAlias.Cell}); err != nil { + return vterrors.Wrap(err, "InitTablet failed to RebuildSrvVSchema") + } + } + case topo.IsErrType(err, topo.NoNode): + // There is no SrvSchema in this cell at all, so we definitely need to rebuild. + if err := agent.TopoServer.RebuildSrvVSchema(ctx, []string{agent.TabletAlias.Cell}); err != nil { + return vterrors.Wrap(err, "InitTablet failed to RebuildSrvVSchema") + } + default: + return vterrors.Wrap(err, "InitTablet failed to read SrvVSchema") } log.Infof("Initializing the tablet for type %v", tabletType)