diff --git a/go/cmd/vtadmin/main.go b/go/cmd/vtadmin/main.go index abc7d5b08c7..cd45b715980 100644 --- a/go/cmd/vtadmin/main.go +++ b/go/cmd/vtadmin/main.go @@ -81,6 +81,10 @@ func main() { rootCmd.Flags().Var(&clusterFileConfig, "cluster-config", "path to a yaml cluster configuration. see clusters.example.yaml") // (TODO:@amason) provide example config. rootCmd.Flags().Var(&defaultClusterConfig, "cluster-defaults", "default options for all clusters") + rootCmd.Flags().AddGoFlag(flag.Lookup("tracer")) // defined in go/vt/trace + rootCmd.Flags().AddGoFlag(flag.Lookup("tracing-enable-logging")) // defined in go/vt/trace + rootCmd.Flags().AddGoFlag(flag.Lookup("tracing-sampling-type")) // defined in go/vt/trace + rootCmd.Flags().AddGoFlag(flag.Lookup("tracing-sampling-rate")) // defined in go/vt/trace rootCmd.Flags().BoolVar(&opts.EnableTracing, "grpc-tracing", false, "whether to enable tracing on the gRPC server") rootCmd.Flags().BoolVar(&httpOpts.EnableTracing, "http-tracing", false, "whether to enable tracing on the HTTP server") rootCmd.Flags().BoolVar(&httpOpts.DisableCompression, "http-no-compress", false, "whether to disable compression of HTTP API responses") diff --git a/go/cmd/vtctlclient/main.go b/go/cmd/vtctlclient/main.go index e7311a5cffa..bd8871851f7 100644 --- a/go/cmd/vtctlclient/main.go +++ b/go/cmd/vtctlclient/main.go @@ -67,6 +67,10 @@ func main() { logutil.LogEvent(logger, e) }) if err != nil { + if strings.Contains(err.Error(), "flag: help requested") { + return + } + errStr := strings.Replace(err.Error(), "remote error: ", "", -1) fmt.Printf("%s Error: %s\n", flag.Arg(0), errStr) log.Error(err) diff --git a/go/cmd/vtctldclient/internal/command/backups.go b/go/cmd/vtctldclient/internal/command/backups.go index 21d5673ef34..38807e3d21a 100644 --- a/go/cmd/vtctldclient/internal/command/backups.go +++ b/go/cmd/vtctldclient/internal/command/backups.go @@ -23,30 +23,54 @@ import ( "github.com/spf13/cobra" "vitess.io/vitess/go/cmd/vtctldclient/cli" + "vitess.io/vitess/go/vt/topo/topoproto" + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" ) // GetBackups makes a GetBackups gRPC call to a vtctld. var GetBackups = &cobra.Command{ - Use: "GetBackups keyspace shard", - Args: cobra.ExactArgs(2), + Use: "GetBackups ", + Args: cobra.ExactArgs(1), RunE: commandGetBackups, } +var getBackupsOptions = struct { + Limit uint32 + Detailed bool + DetailedLimit uint32 + OutputJSON bool +}{} + func commandGetBackups(cmd *cobra.Command, args []string) error { - cli.FinishedParsing(cmd) + keyspace, shard, err := topoproto.ParseKeyspaceShard(cmd.Flags().Arg(0)) + if err != nil { + return err + } - keyspace := cmd.Flags().Arg(0) - shard := cmd.Flags().Arg(1) + cli.FinishedParsing(cmd) resp, err := client.GetBackups(commandCtx, &vtctldatapb.GetBackupsRequest{ - Keyspace: keyspace, - Shard: shard, + Keyspace: keyspace, + Shard: shard, + Limit: getBackupsOptions.Limit, + Detailed: getBackupsOptions.Detailed, + DetailedLimit: getBackupsOptions.DetailedLimit, }) if err != nil { return err } + if getBackupsOptions.OutputJSON || getBackupsOptions.Detailed { + data, err := cli.MarshalJSON(resp) + if err != nil { + return err + } + + fmt.Printf("%s\n", data) + return nil + } + names := make([]string, len(resp.Backups)) for i, b := range resp.Backups { names[i] = b.Name @@ -58,5 +82,9 @@ func commandGetBackups(cmd *cobra.Command, args []string) error { } func init() { + GetBackups.Flags().Uint32VarP(&getBackupsOptions.Limit, "limit", "l", 0, "Retrieve only the most recent N backups") + GetBackups.Flags().BoolVarP(&getBackupsOptions.OutputJSON, "json", "j", false, "Output backup info in JSON format rather than a list of backups") + GetBackups.Flags().BoolVar(&getBackupsOptions.Detailed, "detailed", false, "Get detailed backup info, such as the engine used for each backup, and its status. Implies --json.") + GetBackups.Flags().Uint32Var(&getBackupsOptions.DetailedLimit, "detailed-limit", 0, "Get detailed backup info for only the most recent N backups. Ignored if --detailed is not passed.") Root.AddCommand(GetBackups) } diff --git a/go/cmd/vtctldclient/internal/command/cells.go b/go/cmd/vtctldclient/internal/command/cells.go index e04984f761b..046dde13744 100644 --- a/go/cmd/vtctldclient/internal/command/cells.go +++ b/go/cmd/vtctldclient/internal/command/cells.go @@ -24,10 +24,55 @@ import ( "vitess.io/vitess/go/cmd/vtctldclient/cli" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" ) var ( + // AddCellInfo makes an AddCellInfo gRPC call to a vtctld. + AddCellInfo = &cobra.Command{ + Use: "AddCellInfo --root [--server-address ] ", + Short: "Registers a local topology service in a new cell by creating the CellInfo.", + Long: `Registers a local topology service in a new cell by creating the CellInfo +with the provided parameters. + +The address will be used to connect to the topology service, and Vitess data will +be stored starting at the provided root.`, + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(1), + RunE: commandAddCellInfo, + } + // AddCellsAlias makes an AddCellsAlias gRPC call to a vtctld. + AddCellsAlias = &cobra.Command{ + Use: "AddCellsAlias --cells [--cells ...] ", + Short: "Defines a group of cells that can be referenced by a single name (the alias).", + Long: `Defines a group of cells that can be referenced by a single name (the alias). + +When routing query traffic, replica/rdonly traffic can be routed across cells +within the group (alias). Only primary traffic can be routed across cells not in +the same group (alias).`, + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(1), + RunE: commandAddCellsAlias, + } + // DeleteCellInfo makes a DeleteCellInfo gRPC call to a vtctld. + DeleteCellInfo = &cobra.Command{ + Use: "DeleteCellInfo [--force] ", + Short: "Deletes the CellInfo for the provided cell.", + Long: "Deletes the CellInfo for the provided cell. The cell cannot be referenced by any Shard record.", + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(1), + RunE: commandDeleteCellInfo, + } + // DeleteCellsAlias makes a DeleteCellsAlias gRPC call to a vtctld. + DeleteCellsAlias = &cobra.Command{ + Use: "DeleteCellsAlias ", + Short: "Deletes the CellsAlias for the provided alias.", + Long: "Deletes the CellsAlias for the provided alias.", + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(1), + RunE: commandDeleteCellsAlias, + } // GetCellInfoNames makes a GetCellInfoNames gRPC call to a vtctld. GetCellInfoNames = &cobra.Command{ Use: "GetCellInfoNames", @@ -46,8 +91,99 @@ var ( Args: cobra.NoArgs, RunE: commandGetCellsAliases, } + // UpdateCellInfo makes an UpdateCellInfo gRPC call to a vtctld. + UpdateCellInfo = &cobra.Command{ + Use: "UpdateCellInfo [--root ] [--server-address ] ", + Short: "Updates the content of a CellInfo with the provided parameters, creating the CellInfo if it does not exist.", + Long: `Updates the content of a CellInfo with the provided parameters, creating the CellInfo if it does not exist. + +If a value is empty, it is ignored.`, + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(1), + RunE: commandUpdateCellInfo, + } + // UpdateCellsAlias makes an UpdateCellsAlias gRPC call to a vtctld. + UpdateCellsAlias = &cobra.Command{ + Use: "UpdateCellsAlias [--cells [--cells ...]] ", + Short: "Updates the content of a CellsAlias with the provided parameters, creating the CellsAlias if it does not exist.", + Long: "Updates the content of a CellsAlias with the provided parameters, creating the CellsAlias if it does not exist.", + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(1), + RunE: commandUpdateCellsAlias, + } ) +var addCellInfoOptions topodatapb.CellInfo + +func commandAddCellInfo(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + + cell := cmd.Flags().Arg(0) + _, err := client.AddCellInfo(commandCtx, &vtctldatapb.AddCellInfoRequest{ + Name: cell, + CellInfo: &addCellInfoOptions, + }) + if err != nil { + return err + } + + fmt.Printf("Created cell: %s\n", cell) + return nil +} + +var addCellsAliasOptions topodatapb.CellsAlias + +func commandAddCellsAlias(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + + alias := cmd.Flags().Arg(0) + _, err := client.AddCellsAlias(commandCtx, &vtctldatapb.AddCellsAliasRequest{ + Name: alias, + Cells: addCellsAliasOptions.Cells, + }) + if err != nil { + return err + } + + fmt.Printf("Created cells alias: %s (cells = %v)\n", alias, addCellsAliasOptions.Cells) + return nil +} + +var deleteCellInfoOptions = struct { + Force bool +}{} + +func commandDeleteCellInfo(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + + cell := cmd.Flags().Arg(0) + _, err := client.DeleteCellInfo(commandCtx, &vtctldatapb.DeleteCellInfoRequest{ + Name: cell, + Force: deleteCellInfoOptions.Force, + }) + if err != nil { + return err + } + + fmt.Printf("Deleted cell %s\n", cell) + return nil +} + +func commandDeleteCellsAlias(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + + alias := cmd.Flags().Arg(0) + _, err := client.DeleteCellsAlias(commandCtx, &vtctldatapb.DeleteCellsAliasRequest{ + Name: alias, + }) + if err != nil { + return err + } + + fmt.Printf("Delete cells alias %s\n", alias) + return nil +} + func commandGetCellInfoNames(cmd *cobra.Command, args []string) error { cli.FinishedParsing(cmd) @@ -99,8 +235,73 @@ func commandGetCellsAliases(cmd *cobra.Command, args []string) error { return nil } +var updateCellInfoOptions topodatapb.CellInfo + +func commandUpdateCellInfo(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + + cell := cmd.Flags().Arg(0) + resp, err := client.UpdateCellInfo(commandCtx, &vtctldatapb.UpdateCellInfoRequest{ + Name: cell, + CellInfo: &updateCellInfoOptions, + }) + if err != nil { + return err + } + + data, err := cli.MarshalJSON(resp.CellInfo) + if err != nil { + return err + } + + fmt.Printf("Updated cell %s. New CellInfo:\n%s\n", resp.Name, data) + return nil +} + +var updateCellsAliasOptions topodatapb.CellsAlias + +func commandUpdateCellsAlias(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + + alias := cmd.Flags().Arg(0) + resp, err := client.UpdateCellsAlias(commandCtx, &vtctldatapb.UpdateCellsAliasRequest{ + Name: alias, + CellsAlias: &updateCellsAliasOptions, + }) + if err != nil { + return err + } + + data, err := cli.MarshalJSON(resp.CellsAlias) + if err != nil { + return err + } + + fmt.Printf("Updated cells alias %s. New CellsAlias:\n%s\n", resp.Name, data) + return nil +} + func init() { + AddCellInfo.Flags().StringVarP(&addCellInfoOptions.ServerAddress, "server-address", "a", "", "The address the topology server will connect to for this cell.") + AddCellInfo.Flags().StringVarP(&addCellInfoOptions.Root, "root", "r", "", "The root path the topology server will use for this cell") + AddCellInfo.MarkFlagRequired("root") + Root.AddCommand(AddCellInfo) + + AddCellsAlias.Flags().StringSliceVarP(&addCellsAliasOptions.Cells, "cells", "c", nil, "The list of cell names that are members of this alias.") + Root.AddCommand(AddCellsAlias) + + DeleteCellInfo.Flags().BoolVarP(&deleteCellInfoOptions.Force, "force", "f", false, "Proceeds even if the cell's topology server cannot be reached. The assumption is that you shut down the entire cell, and just need to update the global topo data.") + Root.AddCommand(DeleteCellInfo) + Root.AddCommand(DeleteCellsAlias) + Root.AddCommand(GetCellInfoNames) Root.AddCommand(GetCellInfo) Root.AddCommand(GetCellsAliases) + + UpdateCellInfo.Flags().StringVarP(&updateCellInfoOptions.ServerAddress, "server-address", "a", "", "The address the topology server will connect to for this cell.") + UpdateCellInfo.Flags().StringVarP(&updateCellInfoOptions.Root, "root", "r", "", "The root path the topology server will use for this cell") + Root.AddCommand(UpdateCellInfo) + + UpdateCellsAlias.Flags().StringSliceVarP(&updateCellsAliasOptions.Cells, "cells", "c", nil, "The list of cell names that are members of this alias.") + Root.AddCommand(UpdateCellsAlias) } diff --git a/go/cmd/vtctldclient/internal/command/legacy_shim.go b/go/cmd/vtctldclient/internal/command/legacy_shim.go new file mode 100644 index 00000000000..f4036fd26ae --- /dev/null +++ b/go/cmd/vtctldclient/internal/command/legacy_shim.go @@ -0,0 +1,104 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "context" + "flag" + "fmt" + "strings" + + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/vtctldclient/cli" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/vtctl/vtctlclient" + + logutilpb "vitess.io/vitess/go/vt/proto/logutil" +) + +var ( + // LegacyVtctlCommand provides a shim to make legacy ExecuteVtctlCommand + // RPCs. This allows users to use a single binary to make RPCs against both + // the new and old vtctld gRPC APIs. + LegacyVtctlCommand = &cobra.Command{ + Use: "LegacyVtctlCommand -- [flags ...] [args ...]", + Short: "Invoke a legacy vtctlclient command. Flag parsing is best effort.", + Args: cobra.ArbitraryArgs, + RunE: func(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + return runLegacyCommand(args) + }, + Long: strings.TrimSpace(` +LegacyVtctlCommand uses the legacy vtctl grpc client to make an ExecuteVtctlCommand +rpc to a vtctld. + +This command exists to support a smooth transition of any scripts that relied on +vtctlclient during the migration to the new vtctldclient, and will be removed, +following the Vitess project's standard deprecation cycle, once all commands +have been migrated to the new VtctldServer api. + +To see the list of available legacy commands, run "LegacyVtctlCommand -- help". +Note that, as with the old client, this requires a running server, as the flag +parsing and help/usage text generation, is done server-side. + +Also note that, in order to defer that flag parsing to the server side, you must +use the double-dash ("--") after the LegacyVtctlCommand subcommand string, or +the client-side flag parsing library we are using will attempt to parse those +flags (and fail). +`), + Example: strings.TrimSpace(` +LegacyVtctlCommand help # displays this help message +LegacyVtctlCommand -- help # displays help for supported legacy vtctl commands + +# When using legacy command that take arguments, a double dash must be used +# before the first flag argument, like in the first example. The double dash may +# be used, however, at any point after the "LegacyVtctlCommand" string, as in +# the second example. +LegacyVtctlCommand AddCellInfo -- -server_address "localhost:1234" -root "/vitess/cell1" +LegacyVtctlCommand -- AddCellInfo -server_address "localhost:5678" -root "/vitess/cell1"`), + } +) + +func runLegacyCommand(args []string) error { + // Duplicated (mostly) from go/cmd/vtctlclient/main.go. + logger := logutil.NewConsoleLogger() + + ctx, cancel := context.WithTimeout(context.Background(), actionTimeout) + defer cancel() + + err := vtctlclient.RunCommandAndWait(ctx, server, args, func(e *logutilpb.Event) { + logutil.LogEvent(logger, e) + }) + if err != nil { + if strings.Contains(err.Error(), "flag: help requested") { + // Help is caught by SetHelpFunc, so we don't want to indicate this as an error. + return nil + } + + errStr := strings.Replace(err.Error(), "remote error: ", "", -1) + fmt.Printf("%s Error: %s\n", flag.Arg(0), errStr) + log.Error(err) + } + + return err +} + +func init() { + Root.AddCommand(LegacyVtctlCommand) +} diff --git a/go/cmd/vtctldclient/internal/command/root.go b/go/cmd/vtctldclient/internal/command/root.go index 7243c8836e5..8335710f449 100644 --- a/go/cmd/vtctldclient/internal/command/root.go +++ b/go/cmd/vtctldclient/internal/command/root.go @@ -25,7 +25,6 @@ import ( "github.com/spf13/cobra" "vitess.io/vitess/go/trace" - "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vtctl/vtctldclient" ) @@ -44,9 +43,7 @@ var ( // command context for every command. PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { traceCloser = trace.StartTracing("vtctldclient") - if server == "" { - err = errors.New("please specify -server to specify the vtctld server to connect to") - log.Error(err) + if err := ensureServerArg(); err != nil { return err } @@ -75,6 +72,17 @@ var ( } ) +var errNoServer = errors.New("please specify -server to specify the vtctld server to connect to") + +// ensureServerArg validates that --server was passed to the CLI. +func ensureServerArg() error { + if server == "" { + return errNoServer + } + + return nil +} + func init() { Root.PersistentFlags().StringVar(&server, "server", "", "server to use for connection") Root.PersistentFlags().DurationVar(&actionTimeout, "action_timeout", time.Hour, "timeout for the total command") diff --git a/go/cmd/vtctldclient/internal/command/routing_rules.go b/go/cmd/vtctldclient/internal/command/routing_rules.go new file mode 100644 index 00000000000..e43aa49c915 --- /dev/null +++ b/go/cmd/vtctldclient/internal/command/routing_rules.go @@ -0,0 +1,157 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "errors" + "fmt" + "io/ioutil" + "strings" + + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/vtctldclient/cli" + "vitess.io/vitess/go/json2" + + vschemapb "vitess.io/vitess/go/vt/proto/vschema" + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" +) + +var ( + // ApplyRoutingRules makes an ApplyRoutingRules gRPC call to a vtctld. + ApplyRoutingRules = &cobra.Command{ + Use: "ApplyRoutingRules {--rules RULES | --rules-file RULES_FILE} [--cells=c1,c2,...] [--skip-rebuild] [--dry-run]", + Short: "Applies the VSchema routing rules.", + DisableFlagsInUseLine: true, + Args: cobra.NoArgs, + RunE: commandApplyRoutingRules, + } + // GetRoutingRules makes a GetRoutingRules gRPC call to a vtctld. + GetRoutingRules = &cobra.Command{ + Use: "GetRoutingRules", + Short: "Displays the VSchema routing rules.", + DisableFlagsInUseLine: true, + Args: cobra.NoArgs, + RunE: commandGetRoutingRules, + } +) + +var applyRoutingRulesOptions = struct { + Rules string + RulesFilePath string + Cells []string + SkipRebuild bool + DryRun bool +}{} + +func commandApplyRoutingRules(cmd *cobra.Command, args []string) error { + if applyRoutingRulesOptions.Rules != "" && applyRoutingRulesOptions.RulesFilePath != "" { + return fmt.Errorf("cannot pass both --rules (=%s) and --rules-file (=%s)", applyRoutingRulesOptions.Rules, applyRoutingRulesOptions.RulesFilePath) + } + + if applyRoutingRulesOptions.Rules == "" && applyRoutingRulesOptions.RulesFilePath == "" { + return errors.New("must pass exactly one of --rules and --rules-file") + } + + cli.FinishedParsing(cmd) + + var rulesBytes []byte + if applyRoutingRulesOptions.RulesFilePath != "" { + data, err := ioutil.ReadFile(applyRoutingRulesOptions.RulesFilePath) + if err != nil { + return err + } + + rulesBytes = data + } else { + rulesBytes = []byte(applyRoutingRulesOptions.Rules) + } + + rr := &vschemapb.RoutingRules{} + if err := json2.Unmarshal(rulesBytes, &rr); err != nil { + return err + } + + // Round-trip so when we display the result it's readable. + data, err := cli.MarshalJSON(rr) + if err != nil { + return err + } + + if applyRoutingRulesOptions.DryRun { + fmt.Printf("[DRY RUN] Would have saved new RoutingRules object:\n%s\n", data) + + if applyRoutingRulesOptions.SkipRebuild { + fmt.Println("[DRY RUN] Would not have rebuilt VSchema graph, would have required operator to run RebuildVSchemaGraph for changes to take effect") + } else { + fmt.Print("[DRY RUN] Would have rebuilt the VSchema graph") + if len(applyRoutingRulesOptions.Cells) == 0 { + fmt.Print(" in all cells\n") + } else { + fmt.Printf(" in the following cells: %s.\n", strings.Join(applyRoutingRulesOptions.Cells, ", ")) + } + } + + return nil + } + + _, err = client.ApplyRoutingRules(commandCtx, &vtctldatapb.ApplyRoutingRulesRequest{ + RoutingRules: rr, + SkipRebuild: applyRoutingRulesOptions.SkipRebuild, + RebuildCells: applyRoutingRulesOptions.Cells, + }) + if err != nil { + return err + } + + fmt.Printf("New RoutingRules object:\n%s\nIf this is not what you expected, check the input data (as JSON parsing will skip unexpected fields).\n", data) + + if applyRoutingRulesOptions.SkipRebuild { + fmt.Println("Skipping rebuild of VSchema graph, will need to run RebuildVSchemaGraph for changes to take effect.") + } + + return nil +} + +func commandGetRoutingRules(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + + resp, err := client.GetRoutingRules(commandCtx, &vtctldatapb.GetRoutingRulesRequest{}) + if err != nil { + return err + } + + data, err := cli.MarshalJSON(resp.RoutingRules) + if err != nil { + return err + } + + fmt.Printf("%s\n", data) + + return nil +} + +func init() { + ApplyRoutingRules.Flags().StringVarP(&applyRoutingRulesOptions.Rules, "rules", "r", "", "Routing rules, specified as a string") + ApplyRoutingRules.Flags().StringVarP(&applyRoutingRulesOptions.RulesFilePath, "rules-file", "f", "", "Path to a file containing routing rules specified as JSON") + ApplyRoutingRules.Flags().StringSliceVarP(&applyRoutingRulesOptions.Cells, "cells", "c", nil, "Limit the VSchema graph rebuildingg to the specified cells. Ignored if --skip-rebuild is specified.") + ApplyRoutingRules.Flags().BoolVar(&applyRoutingRulesOptions.SkipRebuild, "skip-rebuild", false, "Skip rebuilding the SrvVSchema objects.") + ApplyRoutingRules.Flags().BoolVarP(&applyRoutingRulesOptions.DryRun, "dry-run", "d", false, "Load the specified routing rules as a validation step, but do not actually apply the rules to the topo.") + Root.AddCommand(ApplyRoutingRules) + + Root.AddCommand(GetRoutingRules) +} diff --git a/go/cmd/vtctldclient/internal/command/serving_graph.go b/go/cmd/vtctldclient/internal/command/serving_graph.go index 18a5d04ec37..2c7065088cd 100644 --- a/go/cmd/vtctldclient/internal/command/serving_graph.go +++ b/go/cmd/vtctldclient/internal/command/serving_graph.go @@ -29,15 +29,35 @@ import ( var ( // GetSrvKeyspaces makes a GetSrvKeyspaces gRPC call to a vtctld. GetSrvKeyspaces = &cobra.Command{ - Use: "GetSrvKeyspaces [ ...]", - Args: cobra.MinimumNArgs(1), - RunE: commandGetSrvKeyspaces, + Use: "GetSrvKeyspaces [ ...]", + Short: "Returns the SrvKeyspaces for the given keyspace in one or more cells.", + Args: cobra.MinimumNArgs(1), + RunE: commandGetSrvKeyspaces, + DisableFlagsInUseLine: true, } // GetSrvVSchema makes a GetSrvVSchema gRPC call to a vtctld. GetSrvVSchema = &cobra.Command{ - Use: "GetSrvVSchema cell", - Args: cobra.ExactArgs(1), - RunE: commandGetSrvVSchema, + Use: "GetSrvVSchema cell", + Short: "Returns the SrvVSchema for the given cell.", + Args: cobra.ExactArgs(1), + RunE: commandGetSrvVSchema, + DisableFlagsInUseLine: true, + } + // GetSrvVSchemas makes a GetSrvVSchemas gRPC call to a vtctld. + GetSrvVSchemas = &cobra.Command{ + Use: "GetSrvVSchemas [ ...]", + Short: "Returns the SrvVSchema for all cells, optionally filtered by the given cells.", + Args: cobra.ArbitraryArgs, + RunE: commandGetSrvVSchemas, + DisableFlagsInUseLine: true, + } + // RebuildVSchemaGraph makes a RebuildVSchemaGraph gRPC call to a vtctld. + RebuildVSchemaGraph = &cobra.Command{ + Use: "RebuildVSchemaGraph [--cells=c1,c2,...]", + Short: "Rebuilds the cell-specific SrvVSchema from the global VSchema objects in the provided cells (or all cells if none provided).", + DisableFlagsInUseLine: true, + Args: cobra.NoArgs, + RunE: commandRebuildVSchemaGraph, } ) @@ -87,7 +107,57 @@ func commandGetSrvVSchema(cmd *cobra.Command, args []string) error { return nil } +func commandGetSrvVSchemas(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + + cells := cmd.Flags().Args()[0:] + + resp, err := client.GetSrvVSchemas(commandCtx, &vtctldatapb.GetSrvVSchemasRequest{ + Cells: cells, + }) + if err != nil { + return err + } + + // By default, an empty array will serialize as `null`, but `[]` is a little nicer. + data := []byte("[]") + + if len(resp.SrvVSchemas) > 0 { + data, err = cli.MarshalJSON(resp.SrvVSchemas) + if err != nil { + return err + } + } + + fmt.Printf("%s\n", data) + + return nil +} + +var rebuildVSchemaGraphOptions = struct { + Cells []string +}{} + +func commandRebuildVSchemaGraph(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + + _, err := client.RebuildVSchemaGraph(commandCtx, &vtctldatapb.RebuildVSchemaGraphRequest{ + Cells: rebuildVSchemaGraphOptions.Cells, + }) + if err != nil { + return err + } + + fmt.Println("RebuildVSchemaGraph: ok") + + return nil +} + func init() { Root.AddCommand(GetSrvKeyspaces) Root.AddCommand(GetSrvVSchema) + Root.AddCommand(GetSrvVSchemas) + + RebuildVSchemaGraph.Flags().StringSliceVarP(&rebuildVSchemaGraphOptions.Cells, "cells", "c", nil, "Specifies a comma-separated list of cells to look for tablets") + Root.AddCommand(RebuildVSchemaGraph) } diff --git a/go/cmd/vtctldclient/internal/command/tablets.go b/go/cmd/vtctldclient/internal/command/tablets.go index 65aef8298fd..a8ff785a677 100644 --- a/go/cmd/vtctldclient/internal/command/tablets.go +++ b/go/cmd/vtctldclient/internal/command/tablets.go @@ -32,27 +32,69 @@ import ( var ( // ChangeTabletType makes a ChangeTabletType gRPC call to a vtctld. ChangeTabletType = &cobra.Command{ - Use: "ChangeTabletType [--dry-run] TABLET_ALIAS TABLET_TYPE", - Args: cobra.ExactArgs(2), - RunE: commandChangeTabletType, + Use: "ChangeTabletType [--dry-run] ", + Short: "Changes the db type for the specified tablet, if possible.", + Long: `Changes the db type for the specified tablet, if possible. + +This command is used primarily to arrange replicas, and it will not convert a primary. +NOTE: This command automatically updates the serving graph.`, + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(2), + RunE: commandChangeTabletType, } // DeleteTablets makes a DeleteTablets gRPC call to a vtctld. DeleteTablets = &cobra.Command{ - Use: "DeleteTablets TABLET_ALIAS [ TABLET_ALIAS ... ]", - Args: cobra.MinimumNArgs(1), - RunE: commandDeleteTablets, + Use: "DeleteTablets [ ... ]", + Short: "Deletes tablet(s) from the topology.", + DisableFlagsInUseLine: true, + Args: cobra.MinimumNArgs(1), + RunE: commandDeleteTablets, } // GetTablet makes a GetTablet gRPC call to a vtctld. GetTablet = &cobra.Command{ - Use: "GetTablet alias", - Args: cobra.ExactArgs(1), - RunE: commandGetTablet, + Use: "GetTablet ", + Short: "Outputs a JSON structure that contains information about the tablet.", + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(1), + RunE: commandGetTablet, } // GetTablets makes a GetTablets gRPC call to a vtctld. GetTablets = &cobra.Command{ - Use: "GetTablets [--strict] [{--cell $c1 [--cell $c2 ...], --keyspace $ks [--shard $shard], --tablet-alias $alias}]", - Args: cobra.NoArgs, - RunE: commandGetTablets, + Use: "GetTablets [--strict] [{--cell $c1 [--cell $c2 ...], --keyspace $ks [--shard $shard], --tablet-alias $alias}]", + Short: "Looks up tablets according to filter criteria.", + Long: `Looks up tablets according to the filter criteria. + +If --tablet-alias is passed, none of the other filters (keyspace, shard, cell) may +be passed, and tablets are looked up by tablet alias only. + +If --keyspace is passed, then all tablets in the keyspace are retrieved. The +--shard flag may also be passed to further narrow the set of tablets to that +. Passing --shard without also passing --keyspace will fail. + +Passing --cell limits the set of tablets to those in the specified cells. The +--cell flag accepts a CSV argument (e.g. --cell "c1,c2") and may be repeated +(e.g. --cell "c1" --cell "c2"). + +Valid output formats are "awk" and "json".`, + DisableFlagsInUseLine: true, + Args: cobra.NoArgs, + RunE: commandGetTablets, + } + // RefreshState makes a RefreshState gRPC call to a vtctld. + RefreshState = &cobra.Command{ + Use: "RefreshState ", + Short: "Reloads the tablet record on the specified tablet.", + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(1), + RunE: commandRefreshState, + } + // RefreshStateByShard makes a RefreshStateByShard gRPC call to a vtcld. + RefreshStateByShard = &cobra.Command{ + Use: "RefreshStateByShard [--cell ...] ", + Short: "Reloads the tablet record all tablets in the shard, optionally limited to the specified cells.", + DisableFlagsInUseLine: true, + Args: cobra.ExactArgs(1), + RunE: commandRefreshStateByShard, } ) @@ -218,6 +260,60 @@ func commandGetTablets(cmd *cobra.Command, args []string) error { return nil } +func commandRefreshState(cmd *cobra.Command, args []string) error { + alias, err := topoproto.ParseTabletAlias(cmd.Flags().Arg(0)) + if err != nil { + return err + } + + cli.FinishedParsing(cmd) + + _, err = client.RefreshState(commandCtx, &vtctldatapb.RefreshStateRequest{ + TabletAlias: alias, + }) + if err != nil { + return err + } + + fmt.Printf("Refreshed state on %s\n", topoproto.TabletAliasString(alias)) + return nil +} + +var refreshStateByShardOptions = struct { + Cells []string +}{} + +func commandRefreshStateByShard(cmd *cobra.Command, args []string) error { + keyspace, shard, err := topoproto.ParseKeyspaceShard(cmd.Flags().Arg(0)) + if err != nil { + return err + } + + cli.FinishedParsing(cmd) + + resp, err := client.RefreshStateByShard(commandCtx, &vtctldatapb.RefreshStateByShardRequest{ + Keyspace: keyspace, + Shard: shard, + Cells: refreshStateByShardOptions.Cells, + }) + if err != nil { + return err + } + + msg := &strings.Builder{} + msg.WriteString(fmt.Sprintf("Refreshed state on %s/%s", keyspace, shard)) + if len(refreshStateByShardOptions.Cells) > 0 { + msg.WriteString(fmt.Sprintf(" in cells %s", strings.Join(refreshStateByShardOptions.Cells, ", "))) + } + msg.WriteByte('\n') + if resp.IsPartialRefresh { + msg.WriteString("State refresh was partial; some tablets in the shard may not have succeeded.\n") + } + + fmt.Print(msg.String()) + return nil +} + func init() { ChangeTabletType.Flags().BoolVarP(&changeTabletTypeOptions.DryRun, "dry-run", "d", false, "Shows the proposed change without actually executing it") Root.AddCommand(ChangeTabletType) @@ -234,4 +330,9 @@ func init() { GetTablets.Flags().StringVar(&getTabletsOptions.Format, "format", "awk", "Output format to use; valid choices are (json, awk)") GetTablets.Flags().BoolVar(&getTabletsOptions.Strict, "strict", false, "Require all cells to return successful tablet data. Without --strict, tablet listings may be partial.") Root.AddCommand(GetTablets) + + Root.AddCommand(RefreshState) + + RefreshStateByShard.Flags().StringSliceVarP(&refreshStateByShardOptions.Cells, "cells", "c", nil, "If specified, only call RefreshState on tablets in the specified cells. If empty, all cells are considered.") + Root.AddCommand(RefreshStateByShard) } diff --git a/go/cmd/vtctldclient/internal/command/vschemas.go b/go/cmd/vtctldclient/internal/command/vschemas.go index ac4f4499090..5a519b6d9a0 100644 --- a/go/cmd/vtctldclient/internal/command/vschemas.go +++ b/go/cmd/vtctldclient/internal/command/vschemas.go @@ -18,11 +18,14 @@ package command import ( "fmt" + "io/ioutil" "github.com/spf13/cobra" "vitess.io/vitess/go/cmd/vtctldclient/cli" + "vitess.io/vitess/go/json2" + vschemapb "vitess.io/vitess/go/vt/proto/vschema" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" ) @@ -33,8 +36,89 @@ var ( Args: cobra.ExactArgs(1), RunE: commandGetVSchema, } + // ApplyVSchema makes an ApplyVSchema gRPC call to a vtctld. + ApplyVSchema = &cobra.Command{ + Use: "ApplyVSchema {-vschema= || -vschema-file= || -sql= || -sql-file=} [-cells=c1,c2,...] [-skip-rebuild] [-dry-run] ", + Args: cobra.ExactArgs(1), + DisableFlagsInUseLine: true, + RunE: commandApplyVSchema, + Short: "Applies the VTGate routing schema to the provided keyspace. Shows the result after application.", + } ) +var applyVSchemaOptions = struct { + VSchema string + VSchemaFile string + SQL string + SQLFile string + DryRun bool + SkipRebuild bool + Cells []string +}{} + +func commandApplyVSchema(cmd *cobra.Command, args []string) error { + sqlMode := (applyVSchemaOptions.SQL != "") != (applyVSchemaOptions.SQLFile != "") + jsonMode := (applyVSchemaOptions.VSchema != "") != (applyVSchemaOptions.VSchemaFile != "") + + if sqlMode && jsonMode { + return fmt.Errorf("only one of the sql, sql-file, vschema, or vschema-file flags may be specified when calling the ApplyVSchema command") + } + + if !sqlMode && !jsonMode { + return fmt.Errorf("one of the sql, sql-file, vschema, or vschema-file flags must be specified when calling the ApplyVSchema command") + } + + req := &vtctldatapb.ApplyVSchemaRequest{ + Keyspace: cmd.Flags().Arg(0), + SkipRebuild: applyVSchemaOptions.SkipRebuild, + Cells: applyVSchemaOptions.Cells, + DryRun: applyVSchemaOptions.DryRun, + } + + var err error + if sqlMode { + if applyVSchemaOptions.SQLFile != "" { + sqlBytes, err := ioutil.ReadFile(applyVSchemaOptions.SQLFile) + if err != nil { + return err + } + req.Sql = string(sqlBytes) + } else { + req.Sql = applyVSchemaOptions.SQL + } + } else { // jsonMode + var schema []byte + if applyVSchemaOptions.VSchemaFile != "" { + schema, err = ioutil.ReadFile(applyVSchemaOptions.VSchemaFile) + if err != nil { + return err + } + } else { + schema = []byte(applyVSchemaOptions.VSchema) + } + + var vs *vschemapb.Keyspace + err = json2.Unmarshal(schema, vs) + if err != nil { + return err + } + req.VSchema = vs + } + + cli.FinishedParsing(cmd) + + res, err := client.ApplyVSchema(commandCtx, req) + if err != nil { + return err + } + data, err := cli.MarshalJSON(res.VSchema) + if err != nil { + return err + } + fmt.Printf("New VSchema object:\n%s\nIf this is not what you expected, check the input data (as JSON parsing will skip unexpected fields).\n", data) + return nil +} + func commandGetVSchema(cmd *cobra.Command, args []string) error { cli.FinishedParsing(cmd) @@ -58,5 +142,14 @@ func commandGetVSchema(cmd *cobra.Command, args []string) error { } func init() { + ApplyVSchema.Flags().StringVar(&applyVSchemaOptions.VSchema, "vschema", "", "VSchema") + ApplyVSchema.Flags().StringVar(&applyVSchemaOptions.VSchemaFile, "vschema-file", "", "VSchema File") + ApplyVSchema.Flags().StringVar(&applyVSchemaOptions.SQL, "sql", "", "A VSchema DDL SQL statement, e.g. `alter table t add vindex hash(id)`") + ApplyVSchema.Flags().StringVar(&applyVSchemaOptions.SQLFile, "sql-file", "", "A file containing VSchema DDL SQL") + ApplyVSchema.Flags().BoolVar(&applyVSchemaOptions.DryRun, "dry-run", false, "If set, do not save the altered vschema, simply echo to console.") + ApplyVSchema.Flags().BoolVar(&applyVSchemaOptions.SkipRebuild, "skip-rebuild", false, "If set, do no rebuild the SrvSchema objects.") + ApplyVSchema.Flags().StringSliceVar(&applyVSchemaOptions.Cells, "cells", nil, "If specified, limits the rebuild to the cells, after upload. Ignored if skipRebuild is set.") + Root.AddCommand(ApplyVSchema) + Root.AddCommand(GetVSchema) } diff --git a/go/cmd/vtctldclient/plugin_grpcvtctlclient.go b/go/cmd/vtctldclient/plugin_grpcvtctlclient.go new file mode 100644 index 00000000000..48c631a8baa --- /dev/null +++ b/go/cmd/vtctldclient/plugin_grpcvtctlclient.go @@ -0,0 +1,23 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// Imports and registers the gRPC vtctl client. + +import ( + _ "vitess.io/vitess/go/vt/vtctl/grpcvtctlclient" +) diff --git a/go/flagutil/optional.go b/go/flagutil/optional.go new file mode 100644 index 00000000000..3bfcd3dd473 --- /dev/null +++ b/go/flagutil/optional.go @@ -0,0 +1,146 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flagutil + +import ( + "errors" + "flag" + "strconv" +) + +// OptionalFlag augements the flag.Value interface with a method to determine +// if a flag was set explicitly on the comand-line. +// +// Though not part of the interface, because the return type would be different +// for each implementation, by convention, each implementation should define a +// Get() method to access the underlying value. +type OptionalFlag interface { + flag.Value + IsSet() bool +} + +var ( + _ OptionalFlag = (*OptionalFloat64)(nil) + _ OptionalFlag = (*OptionalString)(nil) +) + +// OptionalFloat64 implements OptionalFlag for float64 values. +type OptionalFloat64 struct { + val float64 + set bool +} + +// NewOptionalFloat64 returns an OptionalFloat64 with the specified value as its +// starting value. +func NewOptionalFloat64(val float64) *OptionalFloat64 { + return &OptionalFloat64{ + val: val, + set: false, + } +} + +// Set is part of the flag.Value interface. +func (f *OptionalFloat64) Set(arg string) error { + v, err := strconv.ParseFloat(arg, 64) + if err != nil { + return numError(err) + } + + f.val = v + f.set = true + + return nil +} + +// String is part of the flag.Value interface. +func (f *OptionalFloat64) String() string { + return strconv.FormatFloat(f.val, 'g', -1, 64) +} + +// Get returns the underlying float64 value of this flag. If the flag was not +// explicitly set, this will be the initial value passed to the constructor. +func (f *OptionalFloat64) Get() float64 { + return f.val +} + +// IsSet is part of the OptionalFlag interface. +func (f *OptionalFloat64) IsSet() bool { + return f.set +} + +// OptionalString implements OptionalFlag for string values. +type OptionalString struct { + val string + set bool +} + +// NewOptionalString returns an OptionalString with the specified value as its +// starting value. +func NewOptionalString(val string) *OptionalString { + return &OptionalString{ + val: val, + set: false, + } +} + +// Set is part of the flag.Value interface. +func (f *OptionalString) Set(arg string) error { + f.val = arg + f.set = true + return nil +} + +// String is part of the flag.Value interface. +func (f *OptionalString) String() string { + return f.val +} + +// Get returns the underlying string value of this flag. If the flag was not +// explicitly set, this will be the initial value passed to the constructor. +func (f *OptionalString) Get() string { + return f.val +} + +// IsSet is part of the OptionalFlag interface. +func (f *OptionalString) IsSet() bool { + return f.set +} + +// lifted directly from package flag to make the behavior of numeric parsing +// consistent with the standard library for our custom optional types. +var ( + errParse = errors.New("parse error") + errRange = errors.New("value out of range") +) + +// lifted directly from package flag to make the behavior of numeric parsing +// consistent with the standard library for our custom optional types. +func numError(err error) error { + ne, ok := err.(*strconv.NumError) + if !ok { + return err + } + + switch ne.Err { + case strconv.ErrSyntax: + return errParse + case strconv.ErrRange: + return errRange + default: + return err + } +} diff --git a/go/protoutil/time.go b/go/protoutil/time.go new file mode 100644 index 00000000000..c226001d4b2 --- /dev/null +++ b/go/protoutil/time.go @@ -0,0 +1,44 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protoutil + +import ( + "time" + + "vitess.io/vitess/go/vt/proto/vttime" +) + +// TimeFromProto converts a vttime.Time proto message into a time.Time object. +func TimeFromProto(tpb *vttime.Time) time.Time { + if tpb == nil { + return time.Time{} + } + + return time.Unix(tpb.Seconds, int64(tpb.Nanoseconds)) +} + +// TimeToProto converts a time.Time object into a vttime.Time proto mesasge. +func TimeToProto(t time.Time) *vttime.Time { + secs, nanos := t.Unix(), t.UnixNano() + + nsecs := secs * 1e9 + extraNanos := nanos - nsecs + return &vttime.Time{ + Seconds: secs, + Nanoseconds: int32(extraNanos), + } +} diff --git a/go/protoutil/time_test.go b/go/protoutil/time_test.go new file mode 100644 index 00000000000..eb3ecb2f0a9 --- /dev/null +++ b/go/protoutil/time_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protoutil + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "vitess.io/vitess/go/test/utils" + "vitess.io/vitess/go/vt/proto/vttime" +) + +func TestTimeFromProto(t *testing.T) { + now := time.Date(2021, time.June, 12, 13, 14, 15, 0 /* nanos */, time.UTC) + vtt := TimeToProto(now) + + utils.MustMatch(t, now, TimeFromProto(vtt)) + + vtt.Nanoseconds = 100 + utils.MustMatch(t, now.Add(100*time.Nanosecond), TimeFromProto(vtt)) + + vtt.Nanoseconds = 1e9 + utils.MustMatch(t, now.Add(time.Second), TimeFromProto(vtt)) + + assert.True(t, TimeFromProto(nil).IsZero(), "expected Go time from nil vttime to be Zero") +} + +func TestTimeToProto(t *testing.T) { + now := time.Date(2021, time.June, 12, 13, 14, 15, 0 /* nanos */, time.UTC) + secs := now.Unix() + utils.MustMatch(t, &vttime.Time{Seconds: secs}, TimeToProto(now)) + + // Testing secs/nanos conversions + utils.MustMatch(t, &vttime.Time{Seconds: secs, Nanoseconds: 100}, TimeToProto(now.Add(100*time.Nanosecond))) + utils.MustMatch(t, &vttime.Time{Seconds: secs + 1}, TimeToProto(now.Add(1e9*time.Nanosecond))) // this should rollover to a full second +} diff --git a/go/trace/logger.go b/go/trace/logger.go new file mode 100644 index 00000000000..158fab3c8b8 --- /dev/null +++ b/go/trace/logger.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trace + +import "vitess.io/vitess/go/vt/log" + +// traceLogger wraps the standard vitess log package to satisfy the datadog and +// jaeger logger interfaces. +type traceLogger struct{} + +// Log is part of the ddtrace.Logger interface. Datadog only ever logs errors. +func (*traceLogger) Log(msg string) { log.Errorf(msg) } + +// Error is part of the jaeger.Logger interface. +func (*traceLogger) Error(msg string) { log.Errorf(msg) } + +// Infof is part of the jaeger.Logger interface. +func (*traceLogger) Infof(msg string, args ...interface{}) { log.Infof(msg, args...) } diff --git a/go/trace/plugin_datadog.go b/go/trace/plugin_datadog.go index 87809d9bc50..f222af1cbeb 100644 --- a/go/trace/plugin_datadog.go +++ b/go/trace/plugin_datadog.go @@ -20,12 +20,18 @@ func newDatadogTracer(serviceName string) (tracingService, io.Closer, error) { return nil, nil, fmt.Errorf("need host and port to datadog agent to use datadog tracing") } - t := opentracer.New( - ddtracer.WithAgentAddr(*dataDogHost+":"+*dataDogPort), + opts := []ddtracer.StartOption{ + ddtracer.WithAgentAddr(*dataDogHost + ":" + *dataDogPort), ddtracer.WithServiceName(serviceName), ddtracer.WithDebugMode(true), - ddtracer.WithSampler(ddtracer.NewRateSampler(*samplingRate)), - ) + ddtracer.WithSampler(ddtracer.NewRateSampler(samplingRate.Get())), + } + + if *enableLogging { + opts = append(opts, ddtracer.WithLogger(&traceLogger{})) + } + + t := opentracer.New(opts...) opentracing.SetGlobalTracer(t) diff --git a/go/trace/plugin_jaeger.go b/go/trace/plugin_jaeger.go index 587f3ac5233..38781ed8d5b 100644 --- a/go/trace/plugin_jaeger.go +++ b/go/trace/plugin_jaeger.go @@ -19,11 +19,13 @@ package trace import ( "flag" "io" + "os" "github.com/opentracing/opentracing-go" "github.com/uber/jaeger-client-go" "github.com/uber/jaeger-client-go/config" + "vitess.io/vitess/go/flagutil" "vitess.io/vitess/go/vt/log" ) @@ -35,9 +37,15 @@ included but nothing Jaeger specific. var ( agentHost = flag.String("jaeger-agent-host", "", "host and port to send spans to. if empty, no tracing will be done") - samplingRate = flag.Float64("tracing-sampling-rate", 0.1, "sampling rate for the probabilistic jaeger sampler") + samplingType = flagutil.NewOptionalString("const") + samplingRate = flagutil.NewOptionalFloat64(0.1) ) +func init() { + flag.Var(samplingType, "tracing-sampling-type", "sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote'") + flag.Var(samplingRate, "tracing-sampling-rate", "sampling rate for the probabilistic jaeger sampler") +} + // newJagerTracerFromEnv will instantiate a tracingService implemented by Jaeger, // taking configuration from environment variables. Available properties are: // JAEGER_SERVICE_NAME -- If this is set, the service name used in code will be ignored and this value used instead @@ -70,13 +78,34 @@ func newJagerTracerFromEnv(serviceName string) (tracingService, io.Closer, error cfg.Reporter.LocalAgentHostPort = *agentHost } log.Infof("Tracing to: %v as %v", cfg.Reporter.LocalAgentHostPort, cfg.ServiceName) - cfg.Sampler = &config.SamplerConfig{ - Type: jaeger.SamplerTypeConst, - Param: *samplingRate, + + if os.Getenv("JAEGER_SAMPLER_PARAM") == "" { + // If the environment variable was not set, we take the flag regardless + // of whether it was explicitly set on the command line. + cfg.Sampler.Param = samplingRate.Get() + } else if samplingRate.IsSet() { + // If the environment variable was set, but the user also explicitly + // passed the command line flag, the flag takes precedence. + cfg.Sampler.Param = samplingRate.Get() + } + + if samplingType.IsSet() { + cfg.Sampler.Type = samplingType.Get() + } else if cfg.Sampler.Type == "" { + log.Infof("-tracing-sampler-type was not set, and JAEGER_SAMPLER_TYPE was not set, defaulting to const sampler") + cfg.Sampler.Type = jaeger.SamplerTypeConst + } + + log.Infof("Tracing sampler type %v (param: %v)", cfg.Sampler.Type, cfg.Sampler.Param) + + var opts []config.Option + if *enableLogging { + opts = append(opts, config.Logger(&traceLogger{})) + } else if cfg.Reporter.LogSpans { + log.Warningf("JAEGER_REPORTER_LOG_SPANS was set, but -tracing-enable-logging was not; spans will not be logged") } - log.Infof("Tracing sampling rate: %v", *samplingRate) - tracer, closer, err := cfg.NewTracer() + tracer, closer, err := cfg.NewTracer(opts...) if err != nil { return nil, &nilCloser{}, err diff --git a/go/trace/trace.go b/go/trace/trace.go index 0038051468a..181d3964e57 100644 --- a/go/trace/trace.go +++ b/go/trace/trace.go @@ -133,6 +133,7 @@ var currentTracer tracingService = noopTracingServer{} var ( tracingServer = flag.String("tracer", "noop", "tracing service to use") + enableLogging = flag.Bool("tracing-enable-logging", false, "whether to enable logging in the tracing service") ) // StartTracing enables tracing for a named service diff --git a/go/vt/grpcclient/client.go b/go/vt/grpcclient/client.go index b000a542a41..6ad54ae8dea 100644 --- a/go/vt/grpcclient/client.go +++ b/go/vt/grpcclient/client.go @@ -19,6 +19,7 @@ limitations under the License. package grpcclient import ( + "context" "flag" "time" @@ -58,6 +59,16 @@ func RegisterGRPCDialOptions(grpcDialOptionsFunc func(opts []grpc.DialOption) ([ // failFast is a non-optional parameter because callers are required to specify // what that should be. func Dial(target string, failFast FailFast, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + return DialContext(context.Background(), target, failFast, opts...) +} + +// DialContext creates a grpc connection to the given target. Setup steps are +// covered by the context deadline, and, if WithBlock is specified in the dial +// options, connection establishment steps are covered by the context as well. +// +// failFast is a non-optional parameter because callers are required to specify +// what that should be. +func DialContext(ctx context.Context, target string, failFast FailFast, opts ...grpc.DialOption) (*grpc.ClientConn, error) { grpccommon.EnableTracingOpt() newopts := []grpc.DialOption{ grpc.WithDefaultCallOptions( @@ -98,7 +109,7 @@ func Dial(target string, failFast FailFast, opts ...grpc.DialOption) (*grpc.Clie newopts = append(newopts, interceptors()...) - return grpc.Dial(target, newopts...) + return grpc.DialContext(ctx, target, newopts...) } func interceptors() []grpc.DialOption { diff --git a/go/vt/key/key.go b/go/vt/key/key.go index 9cedad6f409..6a1f986c7fd 100644 --- a/go/vt/key/key.go +++ b/go/vt/key/key.go @@ -20,6 +20,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "errors" "fmt" "math" "regexp" @@ -191,8 +192,30 @@ func KeyRangeEqual(left, right *topodatapb.KeyRange) bool { if right == nil { return len(left.Start) == 0 && len(left.End) == 0 } - return bytes.Equal(left.Start, right.Start) && - bytes.Equal(left.End, right.End) + return bytes.Equal(addPadding(left.Start), addPadding(right.Start)) && + bytes.Equal(addPadding(left.End), addPadding(right.End)) +} + +// addPadding adds padding to make sure keyrange represents an 8 byte integer. +// From Vitess docs: +// A hash vindex produces an 8-byte number. +// This means that all numbers less than 0x8000000000000000 will fall in shard -80. +// Any number with the highest bit set will be >= 0x8000000000000000, and will therefore +// belong to shard 80-. +// This means that from a keyrange perspective -80 == 00-80 == 0000-8000 == 000000-800000 +// If we don't add this padding, we could run into issues when transitioning from keyranges +// that use 2 bytes to 4 bytes. +func addPadding(kr []byte) []byte { + paddedKr := make([]byte, 8) + + for i := 0; i < len(kr); i++ { + paddedKr = append(paddedKr, kr[i]) + } + + for i := len(kr); i < 8; i++ { + paddedKr = append(paddedKr, 0) + } + return paddedKr } // KeyRangeStartSmaller returns true if right's keyrange start is _after_ left's start @@ -214,7 +237,19 @@ func KeyRangeStartEqual(left, right *topodatapb.KeyRange) bool { if right == nil { return len(left.Start) == 0 } - return bytes.Equal(left.Start, right.Start) + return bytes.Equal(addPadding(left.Start), addPadding(right.Start)) +} + +// KeyRangeContiguous returns true if the end of the left key range exactly +// matches the start of the right key range (i.e they are contigious) +func KeyRangeContiguous(left, right *topodatapb.KeyRange) bool { + if left == nil { + return right == nil || (len(right.Start) == 0 && len(right.End) == 0) + } + if right == nil { + return len(left.Start) == 0 && len(left.End) == 0 + } + return bytes.Equal(addPadding(left.End), addPadding(right.Start)) } // KeyRangeEndEqual returns true if both key ranges have the same end @@ -225,7 +260,7 @@ func KeyRangeEndEqual(left, right *topodatapb.KeyRange) bool { if right == nil { return len(left.End) == 0 } - return bytes.Equal(left.End, right.End) + return bytes.Equal(addPadding(left.End), addPadding(right.End)) } // For more info on the following functions, see: @@ -346,3 +381,74 @@ var krRegexp = regexp.MustCompile(`^[0-9a-fA-F]*-[0-9a-fA-F]*$`) func IsKeyRange(kr string) bool { return krRegexp.MatchString(kr) } + +// GenerateShardRanges returns shard ranges assuming a keyspace with N shards. +func GenerateShardRanges(shards int) ([]string, error) { + var format string + var maxShards int + + switch { + case shards <= 0: + return nil, errors.New("shards must be greater than zero") + case shards <= 256: + format = "%02x" + maxShards = 256 + case shards <= 65536: + format = "%04x" + maxShards = 65536 + default: + return nil, errors.New("this function does not support more than 65336 shards in a single keyspace") + } + + rangeFormatter := func(start, end int) string { + var ( + startKid string + endKid string + ) + + if start != 0 { + startKid = fmt.Sprintf(format, start) + } + + if end != maxShards { + endKid = fmt.Sprintf(format, end) + } + + return fmt.Sprintf("%s-%s", startKid, endKid) + } + + start := 0 + end := 0 + + // If shards does not divide evenly into maxShards, then there is some lossiness, + // where each shard is smaller than it should technically be (if, for example, size == 25.6). + // If we choose to keep everything in ints, then we have two choices: + // - Have every shard in #numshards be a uniform size, tack on an additional shard + // at the end of the range to account for the loss. This is bad because if you ask for + // 7 shards, you'll actually get 7 uniform shards with 1 small shard, for 8 total shards. + // It's also bad because one shard will have much different data distribution than the rest. + // - Expand the final shard to include whatever is left in the keyrange. This will give the + // correct number of shards, which is good, but depending on how lossy each individual shard is, + // you could end with that final shard being significantly larger than the rest of the shards, + // so this doesn't solve the data distribution problem. + // + // By tracking the "real" end (both in the real number sense, and in the truthfulness of the value sense), + // we can re-truncate the integer end on each iteration, which spreads the lossiness more + // evenly across the shards. + // + // This implementation has no impact on shard numbers that are powers of 2, even at large numbers, + // which you can see in the tests. + size := float64(maxShards) / float64(shards) + realEnd := float64(0) + shardRanges := make([]string, 0, shards) + + for i := 1; i <= shards; i++ { + realEnd = float64(i) * size + + end = int(realEnd) + shardRanges = append(shardRanges, rangeFormatter(start, end)) + start = end + } + + return shardRanges, nil +} diff --git a/go/vt/key/key_test.go b/go/vt/key/key_test.go index 8643d0bcc73..4153c927e7b 100644 --- a/go/vt/key/key_test.go +++ b/go/vt/key/key_test.go @@ -23,6 +23,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) @@ -211,20 +212,6 @@ func TestKeyRangeAdd(t *testing.T) { out: "40-c0", ok: true, }} - stringToKeyRange := func(spec string) *topodatapb.KeyRange { - if spec == "" { - return nil - } - parts := strings.Split(spec, "-") - if len(parts) != 2 { - panic("invalid spec") - } - kr, err := ParseKeyRangeParts(parts[0], parts[1]) - if err != nil { - panic(err) - } - return kr - } keyRangeToString := func(kr *topodatapb.KeyRange) string { if kr == nil { return "" @@ -240,6 +227,170 @@ func TestKeyRangeAdd(t *testing.T) { } } +func TestKeyRangeEndEqual(t *testing.T) { + testcases := []struct { + first string + second string + out bool + }{{ + first: "", + second: "", + out: true, + }, { + first: "", + second: "-80", + out: false, + }, { + first: "40-", + second: "10-", + out: true, + }, { + first: "-8000", + second: "-80", + out: true, + }, { + first: "-8000", + second: "-8000000000000000", + out: true, + }, { + first: "-80", + second: "-8000", + out: true, + }} + + for _, tcase := range testcases { + first := stringToKeyRange(tcase.first) + second := stringToKeyRange(tcase.second) + out := KeyRangeEndEqual(first, second) + if out != tcase.out { + t.Fatalf("KeyRangeEndEqual(%q, %q) expected %t, got %t", tcase.first, tcase.second, tcase.out, out) + } + } +} + +func TestKeyRangeStartEqual(t *testing.T) { + testcases := []struct { + first string + second string + out bool + }{{ + first: "", + second: "", + out: true, + }, { + first: "", + second: "-80", + out: true, + }, { + first: "40-", + second: "20-", + out: false, + }, { + first: "-8000", + second: "-80", + out: true, + }, { + first: "-8000", + second: "-8000000000000000", + out: true, + }, { + first: "-80", + second: "-8000", + out: true, + }} + + for _, tcase := range testcases { + first := stringToKeyRange(tcase.first) + second := stringToKeyRange(tcase.second) + out := KeyRangeStartEqual(first, second) + if out != tcase.out { + t.Fatalf("KeyRangeStartEqual(%q, %q) expected %t, got %t", tcase.first, tcase.second, tcase.out, out) + } + } +} + +func TestKeyRangeEqual(t *testing.T) { + testcases := []struct { + first string + second string + out bool + }{{ + first: "", + second: "", + out: true, + }, { + first: "", + second: "-80", + out: false, + }, { + first: "-8000", + second: "-80", + out: true, + }, { + first: "-8000", + second: "-8000000000000000", + out: true, + }, { + first: "-80", + second: "-8000", + out: true, + }} + + for _, tcase := range testcases { + first := stringToKeyRange(tcase.first) + second := stringToKeyRange(tcase.second) + out := KeyRangeEqual(first, second) + if out != tcase.out { + t.Fatalf("KeyRangeEqual(%q, %q) expected %t, got %t", tcase.first, tcase.second, tcase.out, out) + } + } +} + +func TestKeyRangeContiguous(t *testing.T) { + testcases := []struct { + first string + second string + out bool + }{{ + first: "-40", + second: "40-80", + out: true, + }, { + first: "40-80", + second: "-40", + out: false, + }, { + first: "-", + second: "-40", + out: true, + }, { + first: "40-80", + second: "c0-", + out: false, + }, { + first: "40-80", + second: "80-c0", + out: true, + }, { + first: "40-80", + second: "8000000000000000-c000000000000000", + out: true, + }, { + first: "4000000000000000-8000000000000000", + second: "80-c0", + out: true, + }} + + for _, tcase := range testcases { + first := stringToKeyRange(tcase.first) + second := stringToKeyRange(tcase.second) + out := KeyRangeContiguous(first, second) + if out != tcase.out { + t.Fatalf("KeyRangeContiguous(%q, %q) expected %t, got %t", tcase.first, tcase.second, tcase.out, out) + } + } +} + func TestEvenShardsKeyRange_Error(t *testing.T) { testCases := []struct { i, n int @@ -579,3 +730,90 @@ func TestIsKeyRange(t *testing.T) { assert.Equal(t, IsKeyRange(tcase.in), tcase.out, tcase.in) } } + +func TestGenerateShardRanges(t *testing.T) { + type args struct { + shards int + } + + tests := []struct { + name string + args args + want []string + wantErr bool + }{ + { + "errors for shards less than 0", + args{0}, + nil, + true, + }, + { + "errors for shards more than 65536", + args{65537}, + nil, + true, + }, + { + "works for a single shard", + args{1}, + []string{"-"}, + false, + }, + { + "works for more than one shard", + args{2}, + []string{"-80", "80-"}, + false, + }, + { + "works for an odd number of shards", + args{7}, + []string{"-24", "24-49", "49-6d", "6d-92", "92-b6", "b6-db", "db-"}, + false, + }, + { + "works for large number of shards", + args{256}, + []string{"-01", "01-02", "02-03", "03-04", "04-05", "05-06", "06-07", "07-08", "08-09", "09-0a", "0a-0b", "0b-0c", "0c-0d", "0d-0e", "0e-0f", "0f-10", "10-11", "11-12", "12-13", "13-14", "14-15", "15-16", "16-17", "17-18", "18-19", "19-1a", "1a-1b", "1b-1c", "1c-1d", "1d-1e", "1e-1f", "1f-20", "20-21", "21-22", "22-23", "23-24", "24-25", "25-26", "26-27", "27-28", "28-29", "29-2a", "2a-2b", "2b-2c", "2c-2d", "2d-2e", "2e-2f", "2f-30", "30-31", "31-32", "32-33", "33-34", "34-35", "35-36", "36-37", "37-38", "38-39", "39-3a", "3a-3b", "3b-3c", "3c-3d", "3d-3e", "3e-3f", "3f-40", "40-41", "41-42", "42-43", "43-44", "44-45", "45-46", "46-47", "47-48", "48-49", "49-4a", "4a-4b", "4b-4c", "4c-4d", "4d-4e", "4e-4f", "4f-50", "50-51", "51-52", "52-53", "53-54", "54-55", "55-56", "56-57", "57-58", "58-59", "59-5a", "5a-5b", "5b-5c", "5c-5d", "5d-5e", "5e-5f", "5f-60", "60-61", "61-62", "62-63", "63-64", "64-65", "65-66", "66-67", "67-68", "68-69", "69-6a", "6a-6b", "6b-6c", "6c-6d", "6d-6e", "6e-6f", "6f-70", "70-71", "71-72", "72-73", "73-74", "74-75", "75-76", "76-77", "77-78", "78-79", "79-7a", "7a-7b", "7b-7c", "7c-7d", "7d-7e", "7e-7f", "7f-80", "80-81", "81-82", "82-83", "83-84", "84-85", "85-86", "86-87", "87-88", "88-89", "89-8a", "8a-8b", "8b-8c", "8c-8d", "8d-8e", "8e-8f", "8f-90", "90-91", "91-92", "92-93", "93-94", "94-95", "95-96", "96-97", "97-98", "98-99", "99-9a", "9a-9b", "9b-9c", "9c-9d", "9d-9e", "9e-9f", "9f-a0", "a0-a1", "a1-a2", "a2-a3", "a3-a4", "a4-a5", "a5-a6", "a6-a7", "a7-a8", "a8-a9", "a9-aa", "aa-ab", "ab-ac", "ac-ad", "ad-ae", "ae-af", "af-b0", "b0-b1", "b1-b2", "b2-b3", "b3-b4", "b4-b5", "b5-b6", "b6-b7", "b7-b8", "b8-b9", "b9-ba", "ba-bb", "bb-bc", "bc-bd", "bd-be", "be-bf", "bf-c0", "c0-c1", "c1-c2", "c2-c3", "c3-c4", "c4-c5", "c5-c6", "c6-c7", "c7-c8", "c8-c9", "c9-ca", "ca-cb", "cb-cc", "cc-cd", "cd-ce", "ce-cf", "cf-d0", "d0-d1", "d1-d2", "d2-d3", "d3-d4", "d4-d5", "d5-d6", "d6-d7", "d7-d8", "d8-d9", "d9-da", "da-db", "db-dc", "dc-dd", "dd-de", "de-df", "df-e0", "e0-e1", "e1-e2", "e2-e3", "e3-e4", "e4-e5", "e5-e6", "e6-e7", "e7-e8", "e8-e9", "e9-ea", "ea-eb", "eb-ec", "ec-ed", "ed-ee", "ee-ef", "ef-f0", "f0-f1", "f1-f2", "f2-f3", "f3-f4", "f4-f5", "f5-f6", "f6-f7", "f7-f8", "f8-f9", "f9-fa", "fa-fb", "fb-fc", "fc-fd", "fd-fe", "fe-ff", "ff-"}, + false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GenerateShardRanges(tt.args.shards) + if tt.wantErr { + assert.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, got, tt.want) + }) + } +} + +func TestShardCalculatorForShardsGreaterThan512(t *testing.T) { + got, err := GenerateShardRanges(512) + assert.NoError(t, err) + + want := "ff80-" + + assert.Equal(t, want, got[511], "Invalid mapping for a 512-shard keyspace. Expected %v, got %v", want, got[511]) +} + +func stringToKeyRange(spec string) *topodatapb.KeyRange { + if spec == "" { + return nil + } + parts := strings.Split(spec, "-") + if len(parts) != 2 { + panic("invalid spec") + } + kr, err := ParseKeyRangeParts(parts[0], parts[1]) + if err != nil { + panic(err) + } + return kr +} diff --git a/go/vt/mysqlctl/azblobbackupstorage/azblob.go b/go/vt/mysqlctl/azblobbackupstorage/azblob.go index 46ff6ca1d79..4573449adf1 100644 --- a/go/vt/mysqlctl/azblobbackupstorage/azblob.go +++ b/go/vt/mysqlctl/azblobbackupstorage/azblob.go @@ -262,6 +262,11 @@ func (bh *AZBlobBackupHandle) ReadFile(ctx context.Context, filename string) (io }), nil } +// CheckFile is part of the BackupHandle interface. It is currently unimplemented. +func (bh *AZBlobBackupHandle) CheckFile(ctx context.Context, filename string) (bool, error) { + return false, nil +} + // AZBlobBackupStorage structs implements the BackupStorage interface for AZBlob type AZBlobBackupStorage struct { } diff --git a/go/vt/mysqlctl/backup.go b/go/vt/mysqlctl/backup.go index e8df43cf8af..c45de9ece41 100644 --- a/go/vt/mysqlctl/backup.go +++ b/go/vt/mysqlctl/backup.go @@ -17,9 +17,11 @@ limitations under the License. package mysqlctl import ( + "encoding/json" "errors" "flag" "fmt" + "io/ioutil" "os" "path/filepath" "strings" @@ -32,7 +34,11 @@ import ( "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vterrors" + + mysqlctlpb "vitess.io/vitess/go/vt/proto/mysqlctl" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) // This file handles the backup and restore related code @@ -138,6 +144,86 @@ func Backup(ctx context.Context, params BackupParams) error { return finishErr } +// GetBackupInfo returns the name of the backupengine used to produce a given +// backup, based on the MANIFEST file from the backup, and the Status of the +// backup, based on the engine-specific definition of what makes a complete or +// valid backup. +func GetBackupInfo(ctx context.Context, bh backupstorage.BackupHandle) (engine string, status mysqlctlpb.BackupInfo_Status, err error) { + mfest, err := bh.ReadFile(ctx, backupManifestFileName) + if err != nil { + // (TODO|@ajm88): extend (backupstorage.BackupHandle).ReadFile to wrap + // certain errors as fs.ErrNotExist, and distinguish between INCOMPLETE + // (MANIFEST has not been written to storage) and INVALID (MANIFEST + // exists but can't be read/parsed). + return "", mysqlctlpb.BackupInfo_INCOMPLETE, err + } + defer mfest.Close() + + mfestBytes, err := ioutil.ReadAll(mfest) + if err != nil { + return "", mysqlctlpb.BackupInfo_INVALID, err + } + + // We unmarshal into a map here rather than using the GetBackupManifest + // because we are going to pass the raw mfestBytes to the particular + // backupengine implementation for further unmarshalling and processing. + // + // As a result, some of this code is duplicated with other functions in this + // package, but doing things this way has the benefit of minimizing extra + // calls to backupstorage.BackupHandle methods (which can be network-y and + // slow, or subject to external rate limits, etc). + var manifest map[string]interface{} + if err := json.Unmarshal(mfestBytes, &manifest); err != nil { + return "", mysqlctlpb.BackupInfo_INVALID, err + } + + engine, ok := manifest["BackupMethod"].(string) + if !ok { + return "", mysqlctlpb.BackupInfo_INVALID, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "missing BackupMethod field in MANIFEST") + } + + be, err := getBackupEngine(engine) + if err != nil { + return engine, mysqlctlpb.BackupInfo_COMPLETE, err + } + + status, err = be.GetBackupStatus(ctx, bh, mfestBytes) + return engine, status, err +} + +// ParseBackupName parses the backup name for a given dir/name, according to +// the format generated by mysqlctl.Backup. An error is returned only if the +// backup name does not have the expected number of parts; errors parsing the +// timestamp and tablet alias are logged, and a nil value is returned for those +// fields in case of error. +func ParseBackupName(dir string, name string) (backupTime *time.Time, alias *topodatapb.TabletAlias, err error) { + parts := strings.Split(name, ".") + if len(parts) != 3 { + return nil, nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cannot backup name %s, expected .