Skip to content

add skip flag that can skip comparing source & destination schema when run splitdiff #6477

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sougou merged 1 commit into from
Aug 2, 2020
Merged

add skip flag that can skip comparing source & destination schema when run splitdiff #6477

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 23 additions & 8 deletions go/vt/worker/split_diff.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ type SplitDiffWorker struct {
minHealthyRdonlyTablets int
destinationTabletType topodatapb.TabletType
parallelDiffsCount int
skipVerify bool
cleaner *wrangler.Cleaner

// populated during WorkerStateInit, read-only after that
Expand All @@ -71,7 +72,7 @@ type SplitDiffWorker struct {
}

// NewSplitDiffWorker returns a new SplitDiffWorker object.
func NewSplitDiffWorker(wr *wrangler.Wrangler, cell, keyspace, shard string, sourceUID uint32, excludeTables []string, minHealthyRdonlyTablets, parallelDiffsCount int, tabletType topodatapb.TabletType) Worker {
func NewSplitDiffWorker(wr *wrangler.Wrangler, cell, keyspace, shard string, sourceUID uint32, excludeTables []string, minHealthyRdonlyTablets, parallelDiffsCount int, tabletType topodatapb.TabletType, skipVerify bool) Worker {
return &SplitDiffWorker{
StatusWorker: NewStatusWorker(),
wr: wr,
Expand All @@ -83,6 +84,7 @@ func NewSplitDiffWorker(wr *wrangler.Wrangler, cell, keyspace, shard string, sou
minHealthyRdonlyTablets: minHealthyRdonlyTablets,
destinationTabletType: tabletType,
parallelDiffsCount: parallelDiffsCount,
skipVerify: skipVerify,
cleaner: &wrangler.Cleaner{},
}
}
Expand Down Expand Up @@ -429,13 +431,26 @@ func (sdw *SplitDiffWorker) diff(ctx context.Context) error {
return rec.Error()
}

sdw.wr.Logger().Infof("Diffing the schema...")
rec = &concurrency.AllErrorRecorder{}
tmutils.DiffSchema("destination", sdw.destinationSchemaDefinition, "source", sdw.sourceSchemaDefinition, rec)
if rec.HasErrors() {
sdw.wr.Logger().Warningf("Different schemas: %v", rec.Error().Error())
} else {
sdw.wr.Logger().Infof("Schema match, good.")
// In splitClone state:
// if source destination shard table has column like:
// `object_id` varchar(128) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT
// Then after copy and exec it on destination the table column will turn to like this:
// `object_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT
// In that case,(mysql's behavior) when source has too many tables contains columns as `object_id`.
// there will be too much differ schema fail error out put on vtworkerclient side says:
// remainder of the error is truncated because gRPC has a size limit on errors
// This will obscure the real problem.
// so add this flag assumed people already know the schema does not match and make the process going on
if !sdw.skipVerify {
sdw.wr.Logger().Infof("Diffing the schema...")
rec = &concurrency.AllErrorRecorder{}
tmutils.DiffSchema("destination", sdw.destinationSchemaDefinition, "source", sdw.sourceSchemaDefinition, rec)
if !rec.HasErrors() {
sdw.wr.Logger().Infof("Schema match, good.")
} else {
sdw.wr.Logger().Warningf("Different schemas: %v", rec.Error().Error())
return rec.Error()
}
}

// read the vschema if needed
Expand Down
5 changes: 3 additions & 2 deletions go/vt/worker/split_diff_cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ func commandSplitDiff(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSe
minHealthyRdonlyTablets := subFlags.Int("min_healthy_rdonly_tablets", defaultMinHealthyTablets, "minimum number of healthy RDONLY tablets before taking out one")
destTabletTypeStr := subFlags.String("dest_tablet_type", defaultDestTabletType, "destination tablet type (RDONLY or REPLICA) that will be used to compare the shards")
parallelDiffsCount := subFlags.Int("parallel_diffs_count", defaultParallelDiffsCount, "number of tables to diff in parallel")
skipVerify := subFlags.Bool("skip-verify", false, "skip verification of source and target schema when diff")
if err := subFlags.Parse(args); err != nil {
return nil, err
}
Expand All @@ -108,7 +109,7 @@ func commandSplitDiff(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSe
return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "command SplitDiff invalid dest_tablet_type: %v", destTabletType)
}

return NewSplitDiffWorker(wr, wi.cell, keyspace, shard, uint32(*sourceUID), excludeTableArray, *minHealthyRdonlyTablets, *parallelDiffsCount, topodatapb.TabletType(destTabletType)), nil
return NewSplitDiffWorker(wr, wi.cell, keyspace, shard, uint32(*sourceUID), excludeTableArray, *minHealthyRdonlyTablets, *parallelDiffsCount, topodatapb.TabletType(destTabletType), *skipVerify), nil
}

// shardsWithSources returns all the shards that have SourceShards set
Expand Down Expand Up @@ -226,7 +227,7 @@ func interactiveSplitDiff(ctx context.Context, wi *Instance, wr *wrangler.Wrangl

// start the diff job
// TODO: @rafael - Add option to set destination tablet type in UI form.
wrk := NewSplitDiffWorker(wr, wi.cell, keyspace, shard, uint32(sourceUID), excludeTableArray, int(minHealthyRdonlyTablets), int(parallelDiffsCount), topodatapb.TabletType_RDONLY)
wrk := NewSplitDiffWorker(wr, wi.cell, keyspace, shard, uint32(sourceUID), excludeTableArray, int(minHealthyRdonlyTablets), int(parallelDiffsCount), topodatapb.TabletType_RDONLY, false)
return wrk, nil, nil, nil
}

Expand Down