diff --git a/build/bazelutil/nogo_config.json b/build/bazelutil/nogo_config.json index fa3aff35e519..f7f42086cd4a 100644 --- a/build/bazelutil/nogo_config.json +++ b/build/bazelutil/nogo_config.json @@ -67,7 +67,7 @@ }, "fmtsafe": { "exclude_files": { - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger/log\\.go$": "format argument is not a constant expression", + "github.com/cockroachdb/cockroach/pkg/roachprod/logger/log\\.go$": "format argument is not a constant expression", "github.com/cockroachdb/cockroach/pkg/util/log/channels\\.go$": "format argument is not a constant expression" }, "only_files": { diff --git a/pkg/cmd/roachprod/BUILD.bazel b/pkg/cmd/roachprod/BUILD.bazel index 631890229798..21a7e32b366f 100644 --- a/pkg/cmd/roachprod/BUILD.bazel +++ b/pkg/cmd/roachprod/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//pkg/roachprod/config", "//pkg/roachprod/errors", "//pkg/roachprod/install", + "//pkg/roachprod/logger", "//pkg/roachprod/ssh", "//pkg/roachprod/ui", "//pkg/roachprod/vm", diff --git a/pkg/cmd/roachprod/flags.go b/pkg/cmd/roachprod/flags.go index 7fd6227d88a0..37117039b692 100644 --- a/pkg/cmd/roachprod/flags.go +++ b/pkg/cmd/roachprod/flags.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachprod" "github.com/cockroachdb/cockroach/pkg/roachprod/config" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/ssh" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/roachprod/vm/gce" @@ -80,6 +81,8 @@ var ( // hostCluster is used for multi-tenant functionality. hostCluster string + + roachprodLibraryLogger *logger.Logger ) func initFlags() { diff --git a/pkg/cmd/roachprod/main.go b/pkg/cmd/roachprod/main.go index b80a03c21f08..4849832d192f 100644 --- a/pkg/cmd/roachprod/main.go +++ b/pkg/cmd/roachprod/main.go @@ -27,6 +27,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachprod/config" rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/ui" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/errors" @@ -125,7 +126,7 @@ Local Clusters Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) (retErr error) { createVMOpts.ClusterName = args[0] - return roachprod.Create(context.Background(), username, numNodes, createVMOpts, providerOptsContainer) + return roachprod.Create(context.Background(), roachprodLibraryLogger, username, numNodes, createVMOpts, providerOptsContainer) }), } @@ -144,7 +145,7 @@ if the user would like to update the keys on the remote hosts. Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) (retErr error) { - return roachprod.SetupSSH(context.Background(), args[0]) + return roachprod.SetupSSH(context.Background(), roachprodLibraryLogger, args[0]) }), } @@ -165,7 +166,7 @@ directories inside ${HOME}/local directory are removed. `, Args: cobra.ArbitraryArgs, Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.Destroy(destroyAllMine, destroyAllLocal, args...) + return roachprod.Destroy(roachprodLibraryLogger, destroyAllMine, destroyAllLocal, args...) }), } @@ -174,7 +175,7 @@ var cachedHostsCmd = &cobra.Command{ Short: "list all clusters (and optionally their host numbers) from local cache", Args: cobra.NoArgs, Run: wrap(func(cmd *cobra.Command, args []string) error { - roachprod.CachedClusters(func(clusterName string, numVMs int) { + roachprod.CachedClusters(roachprodLibraryLogger, func(clusterName string, numVMs int) { if strings.HasPrefix(clusterName, "teamcity") { return } @@ -242,7 +243,7 @@ hosts file. if listJSON && listDetails { return errors.New("'json' option cannot be combined with 'details' option") } - filteredCloud, err := roachprod.List(listMine, listPattern) + filteredCloud, err := roachprod.List(roachprodLibraryLogger, listMine, listPattern) if err != nil { return err } @@ -268,7 +269,7 @@ hosts file. for _, name := range names { c := filteredCloud.Clusters[name] if listDetails { - c.PrintDetails() + c.PrintDetails(roachprodLibraryLogger) } else { fmt.Fprintf(tw, "%s\t%s\t%d", c.Name, c.Clouds(), len(c.VMs)) if !c.IsLocal() { @@ -314,7 +315,7 @@ var syncCmd = &cobra.Command{ Long: ``, Args: cobra.NoArgs, Run: wrap(func(cmd *cobra.Command, args []string) error { - _, err := roachprod.Sync() + _, err := roachprod.Sync(roachprodLibraryLogger) _ = rootCmd.GenBashCompletionFile(bashCompletion) return err }), @@ -330,7 +331,7 @@ hourly by a cronjob so it is not necessary to run manually. `, Args: cobra.NoArgs, Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.GC(dryrun) + return roachprod.GC(roachprodLibraryLogger, dryrun) }), } @@ -344,7 +345,7 @@ destroyed: `, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.Extend(args[0], extendLifetime) + return roachprod.Extend(roachprodLibraryLogger, args[0], extendLifetime) }), } @@ -396,7 +397,7 @@ cluster setting will be set to its value. install.EnvOption(nodeEnv), install.NumRacksOption(numRacks), } - return roachprod.Start(context.Background(), args[0], startOpts, clusterSettingsOpts...) + return roachprod.Start(context.Background(), roachprodLibraryLogger, args[0], startOpts, clusterSettingsOpts...) }), } @@ -426,7 +427,7 @@ signals. wait = true } stopOpts := roachprod.StopOpts{Wait: wait, ProcessTag: tag, Sig: sig} - return roachprod.Stop(context.Background(), args[0], stopOpts) + return roachprod.Stop(context.Background(), roachprodLibraryLogger, args[0], stopOpts) }), } @@ -469,7 +470,7 @@ environment variables to the cockroach process. install.EnvOption(nodeEnv), install.NumRacksOption(numRacks), } - return roachprod.StartTenant(context.Background(), tenantCluster, hostCluster, startOpts, clusterSettingsOpts...) + return roachprod.StartTenant(context.Background(), roachprodLibraryLogger, tenantCluster, hostCluster, startOpts, clusterSettingsOpts...) }), } @@ -484,7 +485,7 @@ default cluster settings. It's intended to be used in conjunction with `, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.Init(context.Background(), args[0]) + return roachprod.Init(context.Background(), roachprodLibraryLogger, args[0]) }), } @@ -504,7 +505,7 @@ The "status" command outputs the binary and PID for the specified nodes: `, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.Status(context.Background(), args[0], tag) + return roachprod.Status(context.Background(), roachprodLibraryLogger, args[0], tag) }), } @@ -529,7 +530,7 @@ into a single stream. } else { dest = args[0] + ".logs" } - return roachprod.Logs(args[0], dest, username, logsOpts) + return roachprod.Logs(roachprodLibraryLogger, args[0], dest, username, logsOpts) }), } @@ -552,7 +553,7 @@ of nodes, outputting a line whenever a change is detected: `, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - messages, err := roachprod.Monitor(context.Background(), args[0], monitorOpts) + messages, err := roachprod.Monitor(context.Background(), roachprodLibraryLogger, args[0], monitorOpts) if err != nil { return err } @@ -581,7 +582,7 @@ nodes. `, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.Wipe(context.Background(), args[0], wipePreserveCerts) + return roachprod.Wipe(context.Background(), roachprodLibraryLogger, args[0], wipePreserveCerts) }), } @@ -611,7 +612,7 @@ the 'zfs rollback' command: Args: cobra.ExactArgs(2), Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.Reformat(context.Background(), args[0], args[1]) + return roachprod.Reformat(context.Background(), roachprodLibraryLogger, args[0], args[1]) }), } @@ -623,7 +624,7 @@ var runCmd = &cobra.Command{ `, Args: cobra.MinimumNArgs(1), Run: wrap(func(_ *cobra.Command, args []string) error { - return roachprod.Run(context.Background(), args[0], extraSSHOptions, tag, secure, os.Stdout, os.Stderr, args[1:]) + return roachprod.Run(context.Background(), roachprodLibraryLogger, args[0], extraSSHOptions, tag, secure, os.Stdout, os.Stderr, args[1:]) }), } @@ -634,7 +635,7 @@ var resetCmd = &cobra.Command{ environments and will fall back to a no-op.`, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) (retErr error) { - return roachprod.Reset(args[0]) + return roachprod.Reset(roachprodLibraryLogger, args[0]) }), } @@ -647,7 +648,7 @@ var installCmd = &cobra.Command{ `, Args: cobra.MinimumNArgs(2), Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.Install(context.Background(), args[0], args[1:]) + return roachprod.Install(context.Background(), roachprodLibraryLogger, args[0], args[1:]) }), } @@ -662,7 +663,7 @@ var downloadCmd = &cobra.Command{ if len(args) == 4 { dest = args[3] } - return roachprod.Download(context.Background(), args[0], src, sha, dest) + return roachprod.Download(context.Background(), roachprodLibraryLogger, args[0], src, sha, dest) }), } @@ -684,7 +685,7 @@ Currently available application options are: if len(args) == 2 { versionArg = args[1] } - urls, err := roachprod.StageURL(args[0], versionArg, stageOS) + urls, err := roachprod.StageURL(roachprodLibraryLogger, args[0], versionArg, stageOS) if err != nil { return err } @@ -723,7 +724,7 @@ Some examples of usage: if len(args) == 3 { versionArg = args[2] } - return roachprod.Stage(context.Background(), args[0], stageOS, stageDir, args[1], versionArg) + return roachprod.Stage(context.Background(), roachprodLibraryLogger, args[0], stageOS, stageDir, args[1], versionArg) }), } @@ -737,7 +738,7 @@ start." `, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.DistributeCerts(context.Background(), args[0]) + return roachprod.DistributeCerts(context.Background(), roachprodLibraryLogger, args[0]) }), } @@ -753,7 +754,7 @@ var putCmd = &cobra.Command{ if len(args) == 3 { dest = args[2] } - return roachprod.Put(context.Background(), args[0], src, dest, useTreeDist) + return roachprod.Put(context.Background(), roachprodLibraryLogger, args[0], src, dest, useTreeDist) }), } @@ -770,7 +771,7 @@ multiple nodes the destination file name will be prefixed with the node number. if len(args) == 3 { dest = args[2] } - return roachprod.Get(args[0], src, dest) + return roachprod.Get(roachprodLibraryLogger, args[0], src, dest) }), } @@ -780,7 +781,7 @@ var sqlCmd = &cobra.Command{ Long: "Run `cockroach sql` on a remote cluster.\n", Args: cobra.MinimumNArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.SQL(context.Background(), args[0], secure, args[1:]) + return roachprod.SQL(context.Background(), roachprodLibraryLogger, args[0], secure, args[1:]) }), } @@ -791,7 +792,7 @@ var pgurlCmd = &cobra.Command{ `, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - urls, err := roachprod.PgURL(context.Background(), args[0], pgurlCertsDir, external, secure) + urls, err := roachprod.PgURL(context.Background(), roachprodLibraryLogger, args[0], pgurlCertsDir, external, secure) if err != nil { return err } @@ -822,7 +823,7 @@ Examples: if cmd.CalledAs() == "pprof-heap" { pprofOpts.Heap = true } - return roachprod.Pprof(args[0], pprofOpts) + return roachprod.Pprof(roachprodLibraryLogger, args[0], pprofOpts) }), } @@ -834,7 +835,7 @@ var adminurlCmd = &cobra.Command{ `, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - urls, err := roachprod.AdminURL(args[0], adminurlPath, adminurlIPs, adminurlOpen, secure) + urls, err := roachprod.AdminURL(roachprodLibraryLogger, args[0], adminurlPath, adminurlIPs, adminurlOpen, secure) if err != nil { return err } @@ -852,7 +853,7 @@ var ipCmd = &cobra.Command{ `, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - ips, err := roachprod.IP(context.Background(), args[0], external) + ips, err := roachprod.IP(context.Background(), roachprodLibraryLogger, args[0], external) if err != nil { return err } @@ -867,7 +868,7 @@ var versionCmd = &cobra.Command{ Use: `version`, Short: `print version information`, RunE: func(cmd *cobra.Command, args []string) error { - fmt.Println(roachprod.Version()) + fmt.Println(roachprod.Version(roachprodLibraryLogger)) return nil }, } @@ -885,6 +886,14 @@ var getProvidersCmd = &cobra.Command{ } func main() { + loggerCfg := logger.Config{Stdout: os.Stdout, Stderr: os.Stderr} + var loggerError error + roachprodLibraryLogger, loggerError = loggerCfg.NewLogger("") + if loggerError != nil { + fmt.Fprintf(os.Stderr, "unable to configure logger: %s\n", loggerError) + os.Exit(1) + } + _ = roachprod.InitProviders() providerOptsContainer = vm.CreateProviderOptionsContainer() // The commands are displayed in the order they are added to rootCmd. Note diff --git a/pkg/cmd/roachtest/BUILD.bazel b/pkg/cmd/roachtest/BUILD.bazel index 610be7e50c00..7221b4ba2d55 100644 --- a/pkg/cmd/roachtest/BUILD.bazel +++ b/pkg/cmd/roachtest/BUILD.bazel @@ -18,7 +18,6 @@ go_library( "//pkg/build", "//pkg/cmd/internal/issues", "//pkg/cmd/roachtest/cluster", - "//pkg/cmd/roachtest/logger", "//pkg/cmd/roachtest/option", "//pkg/cmd/roachtest/registry", "//pkg/cmd/roachtest/spec", @@ -28,6 +27,7 @@ go_library( "//pkg/roachprod", "//pkg/roachprod/config", "//pkg/roachprod/install", + "//pkg/roachprod/logger", "//pkg/roachprod/vm", "//pkg/testutils/skip", "//pkg/util/contextutil", @@ -68,11 +68,11 @@ go_test( tags = ["broken_in_bazel"], deps = [ "//pkg/cmd/roachtest/cluster", - "//pkg/cmd/roachtest/logger", "//pkg/cmd/roachtest/option", "//pkg/cmd/roachtest/registry", "//pkg/cmd/roachtest/spec", "//pkg/cmd/roachtest/test", + "//pkg/roachprod/logger", "//pkg/testutils", "//pkg/util/syncutil", "//pkg/util/version", diff --git a/pkg/cmd/roachtest/cluster.go b/pkg/cmd/roachtest/cluster.go index b7c73e79e4ef..acb1466c227b 100644 --- a/pkg/cmd/roachtest/cluster.go +++ b/pkg/cmd/roachtest/cluster.go @@ -36,12 +36,12 @@ import ( "github.com/armon/circbuf" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/util/contextutil" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -361,7 +361,7 @@ func execCmdEx(ctx context.Context, l *logger.Logger, clusterName string, args . } } - err := roachprod.Run(ctx, clusterName, "" /* SSHOptions */, "" /* processTag */, false /* secure */, roachprodRunStdout, roachprodRunStderr, args) + err := roachprod.Run(ctx, l, clusterName, "" /* SSHOptions */, "" /* processTag */, false /* secure */, roachprodRunStdout, roachprodRunStderr, args) closePipes(ctx) wg.Wait() @@ -911,7 +911,7 @@ func (f *clusterFactory) newCluster( l.PrintfCtx(ctx, "Attempting cluster creation (attempt #%d/%d)", i, maxAttempts) createVMOpts.ClusterName = c.name - err = roachprod.Create(ctx, cfg.username, cfg.spec.NodeCount, createVMOpts, providerOptsContainer) + err = roachprod.Create(ctx, l, cfg.username, cfg.spec.NodeCount, createVMOpts, providerOptsContainer) if err == nil { if err := f.r.registerCluster(c); err != nil { return nil, err @@ -991,7 +991,7 @@ func attachToExistingCluster( if !opt.skipStop { c.status("stopping cluster") - if err := c.StopE(ctx, option.DefaultStopOpts(), c.All()); err != nil { + if err := c.StopE(ctx, l, option.DefaultStopOpts(), c.All()); err != nil { return nil, err } if !opt.skipWipe { @@ -1019,7 +1019,9 @@ func (c *clusterImpl) setTest(t test.Test) { // StopCockroachGracefullyOnNode stops a running cockroach instance on the requested // node before a version upgrade. -func (c *clusterImpl) StopCockroachGracefullyOnNode(ctx context.Context, node int) error { +func (c *clusterImpl) StopCockroachGracefullyOnNode( + ctx context.Context, l *logger.Logger, node int, +) error { port := fmt.Sprintf("{pgport:%d}", node) // Note that the following command line needs to run against both v2.1 // and the current branch. Do not change it in a manner that is @@ -1034,7 +1036,7 @@ func (c *clusterImpl) StopCockroachGracefullyOnNode(ctx context.Context, node in // However, it serves as a reminder that `./cockroach quit` doesn't yet // work well enough -- ideally all listeners and engines are closed by // the time it returns to the client. - c.Stop(ctx, option.DefaultStopOpts(), c.Node(node)) + c.Stop(ctx, l, option.DefaultStopOpts(), c.Node(node)) // TODO(tschottdorf): should return an error. I doubt that we want to // call these *testing.T-style methods on goroutines. return nil @@ -1064,7 +1066,7 @@ func (c *clusterImpl) validate( // Perform validation on the existing cluster. c.status("checking that existing cluster matches spec") pattern := "^" + regexp.QuoteMeta(c.name) + "$" - cloudClusters, err := roachprod.List(false /* listMine */, pattern) + cloudClusters, err := roachprod.List(l, false /* listMine */, pattern) if err != nil { return err } @@ -1194,7 +1196,7 @@ func (c *clusterImpl) FetchTimeseriesData(ctx context.Context, t test.Test) erro return contextutil.RunWithTimeout(ctx, "fetch tsdata", 5*time.Minute, func(ctx context.Context) error { node := 1 for ; node <= c.spec.NodeCount; node++ { - db, err := c.ConnE(ctx, node) + db, err := c.ConnE(ctx, t.L(), node) if err == nil { err = db.Ping() db.Close() @@ -1219,7 +1221,7 @@ func (c *clusterImpl) FetchTimeseriesData(ctx context.Context, t test.Test) erro ); err != nil { return errors.Wrap(err, "cluster.FetchTimeseriesData") } - db, err := c.ConnE(ctx, node) + db, err := c.ConnE(ctx, t.L(), node) if err != nil { return err } @@ -1304,7 +1306,7 @@ func (c *clusterImpl) FailOnDeadNodes(ctx context.Context, t test.Test) { // Don't hang forever. _ = contextutil.RunWithTimeout(ctx, "detect dead nodes", time.Minute, func(ctx context.Context) error { - messages, err := roachprod.Monitor(ctx, c.name, install.MonitorOpts{OneShot: true, IgnoreEmptyNodes: true}) + messages, err := roachprod.Monitor(ctx, t.L(), c.name, install.MonitorOpts{OneShot: true, IgnoreEmptyNodes: true}) // If there's an error, it means either that the monitor command failed // completely, or that it found a dead node worth complaining about. if err != nil { @@ -1386,7 +1388,7 @@ func (c *clusterImpl) FailOnReplicaDivergence(ctx context.Context, t test.Test) if err := contextutil.RunWithTimeout( ctx, "find live node", 5*time.Second, func(ctx context.Context) error { - db = c.Conn(ctx, i) + db = c.Conn(ctx, t.L(), i) _, err := db.ExecContext(ctx, `;`) return err }, @@ -1618,7 +1620,7 @@ func (c *clusterImpl) doDestroy(ctx context.Context, l *logger.Logger) <-chan st // We use a non-cancelable context for running this command. Once we got // here, the cluster cannot be destroyed again, so we really want this // command to succeed. - if err := roachprod.Destroy(false /* destroyAllMine */, false /* destroyAllLocal */, c.name); err != nil { + if err := roachprod.Destroy(l, false /* destroyAllMine */, false /* destroyAllLocal */, c.name); err != nil { l.ErrorfCtx(ctx, "error destroying cluster %s: %s", c, err) } else { l.PrintfCtx(ctx, "destroying cluster %s... done", c) @@ -1633,7 +1635,7 @@ func (c *clusterImpl) doDestroy(ctx context.Context, l *logger.Logger) <-chan st } else { l.PrintfCtx(ctx, "wiping cluster %s", c) c.status("wiping cluster") - if err := roachprod.Wipe(ctx, c.name, false /* preserveCerts */); err != nil { + if err := roachprod.Wipe(ctx, l, c.name, false /* preserveCerts */); err != nil { l.Errorf("%s", err) } if c.localCertsDir != "" { @@ -1672,7 +1674,7 @@ func (c *clusterImpl) PutE( c.status("uploading file") defer c.status("") - return errors.Wrap(roachprod.Put(ctx, c.MakeNodes(nodes...), src, dest, false /* useTreeDist */), "cluster.PutE") + return errors.Wrap(roachprod.Put(ctx, l, c.MakeNodes(nodes...), src, dest, false /* useTreeDist */), "cluster.PutE") } // PutLibraries inserts all available library files into all nodes on the cluster @@ -1714,7 +1716,7 @@ func (c *clusterImpl) Stage( } c.status("staging binary") defer c.status("") - return errors.Wrap(roachprod.Stage(ctx, c.MakeNodes(opts...), "" /* stageOS */, dir, application, versionOrSHA), "cluster.Stage") + return errors.Wrap(roachprod.Stage(ctx, l, c.MakeNodes(opts...), "" /* stageOS */, dir, application, versionOrSHA), "cluster.Stage") } // Get gets files from remote hosts. @@ -1726,7 +1728,7 @@ func (c *clusterImpl) Get( } c.status(fmt.Sprintf("getting %v", src)) defer c.status("") - return errors.Wrap(roachprod.Get(c.MakeNodes(opts...), src, dest), "cluster.Get") + return errors.Wrap(roachprod.Get(l, c.MakeNodes(opts...), src, dest), "cluster.Get") } // Put a string into the specified file on the remote(s). @@ -1812,6 +1814,7 @@ func (c *clusterImpl) clearStatusForClusterOpt(worker bool) { // nodes indicating a range. func (c *clusterImpl) StartE( ctx context.Context, + l *logger.Logger, startOpts option.StartOpts, settings install.ClusterSettings, opts ...option.Option, @@ -1867,7 +1870,7 @@ func (c *clusterImpl) StartE( install.BinaryOption(settings.Binary), } - if err := roachprod.Start(ctx, c.MakeNodes(opts...), startOpts.RoachprodOpts, clusterSettingsOpts...); err != nil { + if err := roachprod.Start(ctx, l, c.MakeNodes(opts...), startOpts.RoachprodOpts, clusterSettingsOpts...); err != nil { return err } @@ -1903,11 +1906,12 @@ func (c *clusterImpl) StartE( // Start is like StartE() except that it will fatal the test on error. func (c *clusterImpl) Start( ctx context.Context, + l *logger.Logger, startOpts option.StartOpts, settings install.ClusterSettings, opts ...option.Option, ) { - if err := c.StartE(ctx, startOpts, settings, opts...); err != nil { + if err := c.StartE(ctx, l, startOpts, settings, opts...); err != nil { c.t.Fatal(err) } } @@ -1924,29 +1928,31 @@ func envExists(envs []string, prefix string) bool { // StopE cockroach nodes running on a subset of the cluster. See cluster.Start() // for a description of the nodes parameter. func (c *clusterImpl) StopE( - ctx context.Context, stopOpts option.StopOpts, nodes ...option.Option, + ctx context.Context, l *logger.Logger, stopOpts option.StopOpts, nodes ...option.Option, ) error { if ctx.Err() != nil { return errors.Wrap(ctx.Err(), "cluster.StopE") } c.setStatusForClusterOpt("stopping", stopOpts.RoachtestOpts.Worker, nodes...) defer c.clearStatusForClusterOpt(stopOpts.RoachtestOpts.Worker) - return errors.Wrap(roachprod.Stop(ctx, c.MakeNodes(nodes...), stopOpts.RoachprodOpts), "cluster.StopE") + return errors.Wrap(roachprod.Stop(ctx, l, c.MakeNodes(nodes...), stopOpts.RoachprodOpts), "cluster.StopE") } // Stop is like StopE, except instead of returning an error, it does // c.t.Fatal(). c.t needs to be set. -func (c *clusterImpl) Stop(ctx context.Context, stopOpts option.StopOpts, opts ...option.Option) { +func (c *clusterImpl) Stop( + ctx context.Context, l *logger.Logger, stopOpts option.StopOpts, opts ...option.Option, +) { if c.t.Failed() { // If the test has failed, don't try to limp along. return } - if err := c.StopE(ctx, stopOpts, opts...); err != nil { + if err := c.StopE(ctx, l, stopOpts, opts...); err != nil { c.t.Fatal(err) } } -func (c *clusterImpl) Reset(ctx context.Context) error { +func (c *clusterImpl) Reset(ctx context.Context, l *logger.Logger) error { if c.t.Failed() { return errors.New("already failed") } @@ -1955,7 +1961,7 @@ func (c *clusterImpl) Reset(ctx context.Context) error { } c.status("resetting cluster") defer c.status() - return errors.Wrap(roachprod.Reset(c.name), "cluster.Reset") + return errors.Wrap(roachprod.Reset(l, c.name), "cluster.Reset") } // WipeE wipes a subset of the nodes in a cluster. See cluster.Start() for a @@ -1970,7 +1976,7 @@ func (c *clusterImpl) WipeE(ctx context.Context, l *logger.Logger, nodes ...opti } c.setStatusForClusterOpt("wiping", false, nodes...) defer c.clearStatusForClusterOpt(false) - return roachprod.Wipe(ctx, c.MakeNodes(nodes...), false /* preserveCerts */) + return roachprod.Wipe(ctx, l, c.MakeNodes(nodes...), false /* preserveCerts */) } // Wipe is like WipeE, except instead of returning an error, it does @@ -2069,7 +2075,7 @@ func (c *clusterImpl) RunWithDetails( testLogger.Printf("> %s\n", strings.Join(args, " ")) } - results, err := roachprod.RunWithDetails(ctx, c.MakeNodes(nodes), "" /* SSHOptions */, "" /* processTag */, false /* secure */, args) + results, err := roachprod.RunWithDetails(ctx, l, c.MakeNodes(nodes), "" /* SSHOptions */, "" /* processTag */, false /* secure */, args) if err != nil { l.Printf("> result: %+v", err) createFailedFile(physicalFileName) @@ -2098,9 +2104,9 @@ func createFailedFile(logFileName string) { // Reformat the disk on the specified node. func (c *clusterImpl) Reformat( - ctx context.Context, node option.NodeListOption, filesystem string, + ctx context.Context, l *logger.Logger, node option.NodeListOption, filesystem string, ) error { - return roachprod.Reformat(ctx, c.name, filesystem) + return roachprod.Reformat(ctx, l, c.name, filesystem) } // Silence unused warning. @@ -2108,7 +2114,7 @@ var _ = (&clusterImpl{}).Reformat // Install a package in a node func (c *clusterImpl) Install( - ctx context.Context, nodes option.NodeListOption, software ...string, + ctx context.Context, l *logger.Logger, nodes option.NodeListOption, software ...string, ) error { if ctx.Err() != nil { return errors.Wrap(ctx.Err(), "cluster.Install") @@ -2116,7 +2122,7 @@ func (c *clusterImpl) Install( if len(software) == 0 { return errors.New("Error running cluster.Install: no software passed") } - return errors.Wrap(roachprod.Install(ctx, c.MakeNodes(nodes), software), "cluster.Install") + return errors.Wrap(roachprod.Install(ctx, l, c.MakeNodes(nodes), software), "cluster.Install") } var reOnlyAlphanumeric = regexp.MustCompile(`[^a-zA-Z0-9]+`) @@ -2165,9 +2171,9 @@ func (c *clusterImpl) loggerForCmd( // internal IPs and communication from a test driver to nodes in a cluster // should use external IPs. func (c *clusterImpl) pgURLErr( - ctx context.Context, node option.NodeListOption, external bool, + ctx context.Context, l *logger.Logger, node option.NodeListOption, external bool, ) ([]string, error) { - urls, err := roachprod.PgURL(ctx, c.MakeNodes(node), c.localCertsDir, external, c.localCertsDir != "" /* secure */) + urls, err := roachprod.PgURL(ctx, l, c.MakeNodes(node), c.localCertsDir, external, c.localCertsDir != "" /* secure */) if err != nil { return nil, err } @@ -2179,9 +2185,9 @@ func (c *clusterImpl) pgURLErr( // InternalPGUrl returns the internal Postgres endpoint for the specified nodes. func (c *clusterImpl) InternalPGUrl( - ctx context.Context, node option.NodeListOption, + ctx context.Context, l *logger.Logger, node option.NodeListOption, ) ([]string, error) { - return c.pgURLErr(ctx, node, false /* external */) + return c.pgURLErr(ctx, l, node, false /* external */) } // Silence unused warning. @@ -2189,9 +2195,9 @@ var _ = (&clusterImpl{}).InternalPGUrl // ExternalPGUrl returns the external Postgres endpoint for the specified nodes. func (c *clusterImpl) ExternalPGUrl( - ctx context.Context, node option.NodeListOption, + ctx context.Context, l *logger.Logger, node option.NodeListOption, ) ([]string, error) { - return c.pgURLErr(ctx, node, true /* external */) + return c.pgURLErr(ctx, l, node, true /* external */) } func addrToAdminUIAddr(c *clusterImpl, addr string) (string, error) { @@ -2238,10 +2244,10 @@ func addrToHostPort(addr string) (string, int, error) { // InternalAdminUIAddr returns the internal Admin UI address in the form host:port // for the specified node. func (c *clusterImpl) InternalAdminUIAddr( - ctx context.Context, node option.NodeListOption, + ctx context.Context, l *logger.Logger, node option.NodeListOption, ) ([]string, error) { var addrs []string - urls, err := c.InternalAddr(ctx, node) + urls, err := c.InternalAddr(ctx, l, node) if err != nil { return nil, err } @@ -2258,10 +2264,10 @@ func (c *clusterImpl) InternalAdminUIAddr( // ExternalAdminUIAddr returns the internal Admin UI address in the form host:port // for the specified node. func (c *clusterImpl) ExternalAdminUIAddr( - ctx context.Context, node option.NodeListOption, + ctx context.Context, l *logger.Logger, node option.NodeListOption, ) ([]string, error) { var addrs []string - externalAddrs, err := c.ExternalAddr(ctx, node) + externalAddrs, err := c.ExternalAddr(ctx, l, node) if err != nil { return nil, err } @@ -2278,10 +2284,10 @@ func (c *clusterImpl) ExternalAdminUIAddr( // InternalAddr returns the internal address in the form host:port for the // specified nodes. func (c *clusterImpl) InternalAddr( - ctx context.Context, node option.NodeListOption, + ctx context.Context, l *logger.Logger, node option.NodeListOption, ) ([]string, error) { var addrs []string - urls, err := c.pgURLErr(ctx, node, false /* external */) + urls, err := c.pgURLErr(ctx, l, node, false /* external */) if err != nil { return nil, err } @@ -2297,10 +2303,10 @@ func (c *clusterImpl) InternalAddr( // InternalIP returns the internal IP addresses for the specified nodes. func (c *clusterImpl) InternalIP( - ctx context.Context, node option.NodeListOption, + ctx context.Context, l *logger.Logger, node option.NodeListOption, ) ([]string, error) { var ips []string - addrs, err := c.InternalAddr(ctx, node) + addrs, err := c.InternalAddr(ctx, l, node) if err != nil { return nil, err } @@ -2317,10 +2323,10 @@ func (c *clusterImpl) InternalIP( // ExternalAddr returns the external address in the form host:port for the // specified node. func (c *clusterImpl) ExternalAddr( - ctx context.Context, node option.NodeListOption, + ctx context.Context, l *logger.Logger, node option.NodeListOption, ) ([]string, error) { var addrs []string - urls, err := c.pgURLErr(ctx, node, true /* external */) + urls, err := c.pgURLErr(ctx, l, node, true /* external */) if err != nil { return nil, err } @@ -2336,10 +2342,10 @@ func (c *clusterImpl) ExternalAddr( // ExternalIP returns the external IP addresses for the specified node. func (c *clusterImpl) ExternalIP( - ctx context.Context, node option.NodeListOption, + ctx context.Context, l *logger.Logger, node option.NodeListOption, ) ([]string, error) { var ips []string - addrs, err := c.ExternalAddr(ctx, node) + addrs, err := c.ExternalAddr(ctx, l, node) if err != nil { return nil, err } @@ -2357,8 +2363,8 @@ func (c *clusterImpl) ExternalIP( var _ = (&clusterImpl{}).ExternalIP // Conn returns a SQL connection to the specified node. -func (c *clusterImpl) Conn(ctx context.Context, node int) *gosql.DB { - urls, err := c.ExternalPGUrl(ctx, c.Node(node)) +func (c *clusterImpl) Conn(ctx context.Context, l *logger.Logger, node int) *gosql.DB { + urls, err := c.ExternalPGUrl(ctx, l, c.Node(node)) if err != nil { c.t.Fatal(err) } @@ -2370,8 +2376,8 @@ func (c *clusterImpl) Conn(ctx context.Context, node int) *gosql.DB { } // ConnE returns a SQL connection to the specified node. -func (c *clusterImpl) ConnE(ctx context.Context, node int) (*gosql.DB, error) { - urls, err := c.ExternalPGUrl(ctx, c.Node(node)) +func (c *clusterImpl) ConnE(ctx context.Context, l *logger.Logger, node int) (*gosql.DB, error) { + urls, err := c.ExternalPGUrl(ctx, l, c.Node(node)) if err != nil { return nil, err } @@ -2402,7 +2408,7 @@ func (c *clusterImpl) Extend(ctx context.Context, d time.Duration, l *logger.Log return errors.Wrap(ctx.Err(), "cluster.Extend") } l.PrintfCtx(ctx, "extending cluster by %s", d.String()) - if err := roachprod.Extend(c.name, d); err != nil { + if err := roachprod.Extend(l, c.name, d); err != nil { l.PrintfCtx(ctx, "roachprod extend failed: %v", err) return errors.Wrap(err, "roachprod extend failed") } diff --git a/pkg/cmd/roachtest/cluster/BUILD.bazel b/pkg/cmd/roachtest/cluster/BUILD.bazel index 6e6d94e3a763..5e9c10ca7fb3 100644 --- a/pkg/cmd/roachtest/cluster/BUILD.bazel +++ b/pkg/cmd/roachtest/cluster/BUILD.bazel @@ -10,10 +10,10 @@ go_library( importpath = "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster", visibility = ["//visibility:public"], deps = [ - "//pkg/cmd/roachtest/logger", "//pkg/cmd/roachtest/option", "//pkg/cmd/roachtest/spec", "//pkg/roachprod/install", + "//pkg/roachprod/logger", "@com_github_cockroachdb_errors//:errors", ], ) diff --git a/pkg/cmd/roachtest/cluster/cluster_interface.go b/pkg/cmd/roachtest/cluster/cluster_interface.go index a1b81576ca07..b2f04f93c198 100644 --- a/pkg/cmd/roachtest/cluster/cluster_interface.go +++ b/pkg/cmd/roachtest/cluster/cluster_interface.go @@ -15,10 +15,10 @@ import ( gosql "database/sql" "os" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" ) // Cluster is the interface through which a given roachtest interacts with the @@ -49,34 +49,34 @@ type Cluster interface { // Starting and stopping CockroachDB. - StartE(ctx context.Context, startOpts option.StartOpts, settings install.ClusterSettings, opts ...option.Option) error - Start(ctx context.Context, startOpts option.StartOpts, settings install.ClusterSettings, opts ...option.Option) - StopE(ctx context.Context, stopOpts option.StopOpts, opts ...option.Option) error - Stop(ctx context.Context, stopOpts option.StopOpts, opts ...option.Option) - StopCockroachGracefullyOnNode(ctx context.Context, node int) error + StartE(ctx context.Context, l *logger.Logger, startOpts option.StartOpts, settings install.ClusterSettings, opts ...option.Option) error + Start(ctx context.Context, l *logger.Logger, startOpts option.StartOpts, settings install.ClusterSettings, opts ...option.Option) + StopE(ctx context.Context, l *logger.Logger, stopOpts option.StopOpts, opts ...option.Option) error + Stop(ctx context.Context, l *logger.Logger, stopOpts option.StopOpts, opts ...option.Option) + StopCockroachGracefullyOnNode(ctx context.Context, l *logger.Logger, node int) error NewMonitor(context.Context, ...option.Option) Monitor // Hostnames and IP addresses of the nodes. - InternalAddr(ctx context.Context, node option.NodeListOption) ([]string, error) - InternalIP(ctx context.Context, node option.NodeListOption) ([]string, error) - ExternalAddr(ctx context.Context, node option.NodeListOption) ([]string, error) - ExternalIP(ctx context.Context, node option.NodeListOption) ([]string, error) + InternalAddr(ctx context.Context, l *logger.Logger, node option.NodeListOption) ([]string, error) + InternalIP(ctx context.Context, l *logger.Logger, node option.NodeListOption) ([]string, error) + ExternalAddr(ctx context.Context, l *logger.Logger, node option.NodeListOption) ([]string, error) + ExternalIP(ctx context.Context, l *logger.Logger, node option.NodeListOption) ([]string, error) // SQL connection strings. - InternalPGUrl(ctx context.Context, node option.NodeListOption) ([]string, error) - ExternalPGUrl(ctx context.Context, node option.NodeListOption) ([]string, error) + InternalPGUrl(ctx context.Context, l *logger.Logger, node option.NodeListOption) ([]string, error) + ExternalPGUrl(ctx context.Context, l *logger.Logger, node option.NodeListOption) ([]string, error) // SQL clients to nodes. - Conn(ctx context.Context, node int) *gosql.DB - ConnE(ctx context.Context, node int) (*gosql.DB, error) + Conn(ctx context.Context, l *logger.Logger, node int) *gosql.DB + ConnE(ctx context.Context, l *logger.Logger, node int) (*gosql.DB, error) // URLs for the Admin UI. - InternalAdminUIAddr(ctx context.Context, node option.NodeListOption) ([]string, error) - ExternalAdminUIAddr(ctx context.Context, node option.NodeListOption) ([]string, error) + InternalAdminUIAddr(ctx context.Context, l *logger.Logger, node option.NodeListOption) ([]string, error) + ExternalAdminUIAddr(ctx context.Context, l *logger.Logger, node option.NodeListOption) ([]string, error) // Running commands on nodes. @@ -118,10 +118,10 @@ type Cluster interface { // Internal niche tools. - Reset(ctx context.Context) error - Reformat(ctx context.Context, node option.NodeListOption, filesystem string) error + Reset(ctx context.Context, l *logger.Logger) error + Reformat(ctx context.Context, l *logger.Logger, node option.NodeListOption, filesystem string) error Install( - ctx context.Context, nodes option.NodeListOption, software ...string, + ctx context.Context, l *logger.Logger, nodes option.NodeListOption, software ...string, ) error // Methods whose inclusion on this interface is purely historical. diff --git a/pkg/cmd/roachtest/cluster_test.go b/pkg/cmd/roachtest/cluster_test.go index d452ddd361a4..ea76162ff854 100644 --- a/pkg/cmd/roachtest/cluster_test.go +++ b/pkg/cmd/roachtest/cluster_test.go @@ -14,10 +14,10 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" test2 "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/util/version" "github.com/stretchr/testify/assert" ) diff --git a/pkg/cmd/roachtest/main.go b/pkg/cmd/roachtest/main.go index 8f0944054631..0f1b57a9d86b 100644 --- a/pkg/cmd/roachtest/main.go +++ b/pkg/cmd/roachtest/main.go @@ -21,11 +21,11 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/build" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/tests" "github.com/cockroachdb/cockroach/pkg/roachprod" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/errors" diff --git a/pkg/cmd/roachtest/monitor.go b/pkg/cmd/roachtest/monitor.go index 2f95c2ff80f6..758205337714 100644 --- a/pkg/cmd/roachtest/monitor.go +++ b/pkg/cmd/roachtest/monitor.go @@ -18,10 +18,10 @@ import ( "sync/atomic" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/roachprod" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/errors" "golang.org/x/sync/errgroup" ) @@ -180,7 +180,7 @@ func (m *monitorImpl) wait() error { wg.Done() }() - messagesChannel, err := roachprod.Monitor(m.ctx, m.nodes, install.MonitorOpts{}) + messagesChannel, err := roachprod.Monitor(m.ctx, m.l, m.nodes, install.MonitorOpts{}) if err != nil { setErr(errors.Wrap(err, "monitor command failure")) return diff --git a/pkg/cmd/roachtest/prometheus/BUILD.bazel b/pkg/cmd/roachtest/prometheus/BUILD.bazel index faf44c9fdfa9..d88ab76ed04f 100644 --- a/pkg/cmd/roachtest/prometheus/BUILD.bazel +++ b/pkg/cmd/roachtest/prometheus/BUILD.bazel @@ -18,8 +18,8 @@ go_library( importpath = "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/prometheus", visibility = ["//visibility:private"], deps = [ - "//pkg/cmd/roachtest/logger", "//pkg/cmd/roachtest/option", + "//pkg/roachprod/logger", "@in_gopkg_yaml_v2//:yaml_v2", ], ) @@ -30,6 +30,7 @@ go_test( embed = [":with-mocks"], # keep deps = [ "//pkg/cmd/roachtest/option", + "//pkg/roachprod/logger", "@com_github_golang_mock//gomock", "@com_github_stretchr_testify//require", ], diff --git a/pkg/cmd/roachtest/prometheus/prometheus.go b/pkg/cmd/roachtest/prometheus/prometheus.go index 5063f4922359..7f66d806698d 100644 --- a/pkg/cmd/roachtest/prometheus/prometheus.go +++ b/pkg/cmd/roachtest/prometheus/prometheus.go @@ -18,8 +18,8 @@ import ( "os" "time" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "gopkg.in/yaml.v2" ) @@ -46,7 +46,7 @@ type Config struct { // It is abstracted to prevent a circular dependency on roachtest, as Cluster // requires the test interface. type Cluster interface { - ExternalIP(context.Context, option.NodeListOption) ([]string, error) + ExternalIP(context.Context, *logger.Logger, option.NodeListOption) ([]string, error) Get(ctx context.Context, l *logger.Logger, src, dest string, opts ...option.Option) error RunE(ctx context.Context, node option.NodeListOption, args ...string) error PutString( @@ -64,6 +64,7 @@ func Init( ctx context.Context, cfg Config, c Cluster, + l *logger.Logger, repeatFunc func(context.Context, option.NodeListOption, string, ...string) error, ) (*Prometheus, error) { if err := c.RunE( @@ -86,6 +87,7 @@ func Init( yamlCfg, err := makeYAMLConfig( ctx, + l, c, cfg.ScrapeConfigs, ) @@ -148,7 +150,9 @@ const ( ) // makeYAMLConfig creates a prometheus YAML config for the server to use. -func makeYAMLConfig(ctx context.Context, c Cluster, scrapeConfigs []ScrapeConfig) (string, error) { +func makeYAMLConfig( + ctx context.Context, l *logger.Logger, c Cluster, scrapeConfigs []ScrapeConfig, +) (string, error) { type yamlStaticConfig struct { Targets []string } @@ -174,7 +178,7 @@ func makeYAMLConfig(ctx context.Context, c Cluster, scrapeConfigs []ScrapeConfig for _, scrapeConfig := range scrapeConfigs { var targets []string for _, scrapeNode := range scrapeConfig.ScrapeNodes { - ips, err := c.ExternalIP(ctx, scrapeNode.Nodes) + ips, err := c.ExternalIP(ctx, l, scrapeNode.Nodes) if err != nil { return "", err } diff --git a/pkg/cmd/roachtest/prometheus/prometheus_test.go b/pkg/cmd/roachtest/prometheus/prometheus_test.go index 1bfc9828510b..e4b9a5560274 100644 --- a/pkg/cmd/roachtest/prometheus/prometheus_test.go +++ b/pkg/cmd/roachtest/prometheus/prometheus_test.go @@ -12,13 +12,27 @@ package prometheus import ( "context" + "io/ioutil" "testing" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" + logger "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) +func nilLogger() *logger.Logger { + lcfg := logger.Config{ + Stdout: ioutil.Discard, + Stderr: ioutil.Discard, + } + l, err := lcfg.NewLogger("" /* path */) + if err != nil { + panic(err) + } + return l +} + func TestMakeYAMLConfig(t *testing.T) { ctx := context.Background() testCases := []struct { @@ -34,13 +48,13 @@ func TestMakeYAMLConfig(t *testing.T) { mockCluster: func(ctrl *gomock.Controller) Cluster { c := NewMockCluster(ctrl) c.EXPECT(). - ExternalIP(ctx, []int{1}). + ExternalIP(ctx, nilLogger(), []int{1}). Return([]string{"127.0.0.1"}, nil) c.EXPECT(). - ExternalIP(ctx, []int{3, 4, 5}). + ExternalIP(ctx, nilLogger(), []int{3, 4, 5}). Return([]string{"127.0.0.3", "127.0.0.4", "127.0.0.5"}, nil) c.EXPECT(). - ExternalIP(ctx, []int{6}). + ExternalIP(ctx, nilLogger(), []int{6}). Return([]string{"127.0.0.6"}, nil) return c }, @@ -94,13 +108,13 @@ scrape_configs: mockCluster: func(ctrl *gomock.Controller) Cluster { c := NewMockCluster(ctrl) c.EXPECT(). - ExternalIP(ctx, []int{3, 4, 5}). + ExternalIP(ctx, nilLogger(), []int{3, 4, 5}). Return([]string{"127.0.0.3", "127.0.0.4", "127.0.0.5"}, nil) c.EXPECT(). - ExternalIP(ctx, []int{6}). + ExternalIP(ctx, nilLogger(), []int{6}). Return([]string{"127.0.0.6"}, nil) c.EXPECT(). - ExternalIP(ctx, []int{8, 9}). + ExternalIP(ctx, nilLogger(), []int{8, 9}). Return([]string{"127.0.0.8", "127.0.0.9"}, nil) return c }, @@ -152,6 +166,7 @@ scrape_configs: cfg, err := makeYAMLConfig( ctx, + nilLogger(), tc.mockCluster(ctrl), tc.scrapeConfigs, ) diff --git a/pkg/cmd/roachtest/test/BUILD.bazel b/pkg/cmd/roachtest/test/BUILD.bazel index b8b36cc3e3e2..a8758568e176 100644 --- a/pkg/cmd/roachtest/test/BUILD.bazel +++ b/pkg/cmd/roachtest/test/BUILD.bazel @@ -6,7 +6,7 @@ go_library( importpath = "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test", visibility = ["//visibility:public"], deps = [ - "//pkg/cmd/roachtest/logger", + "//pkg/roachprod/logger", "//pkg/util/version", ], ) diff --git a/pkg/cmd/roachtest/test/test_interface.go b/pkg/cmd/roachtest/test/test_interface.go index 9fe49d498a4e..d0ada18dccbb 100644 --- a/pkg/cmd/roachtest/test/test_interface.go +++ b/pkg/cmd/roachtest/test/test_interface.go @@ -11,7 +11,7 @@ package test import ( - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/util/version" ) diff --git a/pkg/cmd/roachtest/test_impl.go b/pkg/cmd/roachtest/test_impl.go index add6f44d8506..3ce2b6816b04 100644 --- a/pkg/cmd/roachtest/test_impl.go +++ b/pkg/cmd/roachtest/test_impl.go @@ -21,8 +21,8 @@ import ( "strings" "time" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/testutils/skip" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" diff --git a/pkg/cmd/roachtest/test_runner.go b/pkg/cmd/roachtest/test_runner.go index f8d44158cf2e..067c1ecd0123 100644 --- a/pkg/cmd/roachtest/test_runner.go +++ b/pkg/cmd/roachtest/test_runner.go @@ -30,13 +30,13 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/cmd/internal/issues" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/internal/team" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/quotapool" @@ -822,8 +822,8 @@ func (r *testRunner) runTest( // Send SIGQUIT to dump stacks (this is how CRDB handles it) followed by SIGKILL. stopOpts := option.DefaultStopOpts() stopOpts.RoachprodOpts.Sig = 3 - _ = c.StopE(innerCtx, stopOpts, c.All()) - _ = c.StopE(innerCtx, option.DefaultStopOpts(), c.All()) + _ = c.StopE(innerCtx, teardownL, stopOpts, c.All()) + _ = c.StopE(innerCtx, teardownL, option.DefaultStopOpts(), c.All()) t.L().PrintfCtx(ctx, "CockroachDB nodes aborted; check the stderr log for goroutine stack traces") cancel() } @@ -1194,7 +1194,7 @@ func (r *testRunner) serveHTTP(wr http.ResponseWriter, req *http.Request) { var clusterName, clusterAdminUIAddr string if w.Cluster() != nil { clusterName = w.Cluster().name - adminUIAddrs, err := w.Cluster().ExternalAdminUIAddr(req.Context(), w.Cluster().Node(1)) + adminUIAddrs, err := w.Cluster().ExternalAdminUIAddr(req.Context(), w.Cluster().l, w.Cluster().Node(1)) if err == nil { clusterAdminUIAddr = adminUIAddrs[0] } diff --git a/pkg/cmd/roachtest/test_test.go b/pkg/cmd/roachtest/test_test.go index 3a7d58d85878..ae69bd73d2d0 100644 --- a/pkg/cmd/roachtest/test_test.go +++ b/pkg/cmd/roachtest/test_test.go @@ -21,10 +21,10 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/cockroach/pkg/util/version" diff --git a/pkg/cmd/roachtest/tests/BUILD.bazel b/pkg/cmd/roachtest/tests/BUILD.bazel index e7a736badc8a..efd91c09e1e8 100644 --- a/pkg/cmd/roachtest/tests/BUILD.bazel +++ b/pkg/cmd/roachtest/tests/BUILD.bazel @@ -159,7 +159,6 @@ go_library( "//pkg/clusterversion", "//pkg/cmd/cmpconn", "//pkg/cmd/roachtest/cluster", - "//pkg/cmd/roachtest/logger", "//pkg/cmd/roachtest/option", "//pkg/cmd/roachtest/prometheus:with-mocks", "//pkg/cmd/roachtest/registry", @@ -172,6 +171,7 @@ go_library( "//pkg/kv", "//pkg/roachpb:with-mocks", "//pkg/roachprod/install", + "//pkg/roachprod/logger", "//pkg/server", "//pkg/server/serverpb", "//pkg/sql/pgwire/pgcode", @@ -229,10 +229,10 @@ go_test( ], embed = [":with-mocks"], # keep deps = [ - "//pkg/cmd/roachtest/logger", "//pkg/cmd/roachtest/option", "//pkg/cmd/roachtest/prometheus:with-mocks", "//pkg/cmd/roachtest/spec", + "//pkg/roachprod/logger", "//pkg/testutils/skip", "//pkg/util/version", "@com_github_golang_mock//gomock", diff --git a/pkg/cmd/roachtest/tests/activerecord.go b/pkg/cmd/roachtest/tests/activerecord.go index 9a1c5333b109..316862c61daf 100644 --- a/pkg/cmd/roachtest/tests/activerecord.go +++ b/pkg/cmd/roachtest/tests/activerecord.go @@ -47,19 +47,19 @@ func registerActiveRecord(r registry.Registry) { if err := c.PutLibraries(ctx, "./lib"); err != nil { t.Fatal(err) } - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } t.Status("creating database used by tests") - db, err := c.ConnE(ctx, node[0]) + db, err := c.ConnE(ctx, t.L(), node[0]) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/allocator.go b/pkg/cmd/roachtest/tests/allocator.go index d5ef164c2766..2ad56814f6f0 100644 --- a/pkg/cmd/roachtest/tests/allocator.go +++ b/pkg/cmd/roachtest/tests/allocator.go @@ -18,12 +18,12 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/errors" ) @@ -34,8 +34,8 @@ func registerAllocator(r registry.Registry) { startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = []string{"--vmodule=store_rebalancer=5,allocator=5,allocator_scorer=5,replicate_queue=5"} - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Range(1, start)) - db := c.Conn(ctx, 1) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(1, start)) + db := c.Conn(ctx, t.L(), 1) defer db.Close() m := c.NewMonitor(ctx, c.Range(1, start)) @@ -51,7 +51,7 @@ func registerAllocator(r registry.Registry) { m.Wait() // Start the remaining nodes to kick off upreplication/rebalancing. - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Range(start+1, c.Spec().NodeCount)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(start+1, c.Spec().NodeCount)) c.Run(ctx, c.Node(1), `./cockroach workload init kv --drop`) for node := 1; node <= c.Spec().NodeCount; node++ { @@ -269,9 +269,9 @@ func runWideReplication(ctx context.Context, t test.Test, c cluster.Cluster) { startOpts.RoachprodOpts.ExtraArgs = []string{"--vmodule=replicate_queue=6"} settings := install.MakeClusterSettings() settings.Env = append(settings.Env, "COCKROACH_SCAN_MAX_IDLE_TIME=5ms") - c.Start(ctx, startOpts, settings, c.All()) + c.Start(ctx, t.L(), startOpts, settings, c.All()) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() zones := func() []string { @@ -336,9 +336,9 @@ func runWideReplication(ctx context.Context, t test.Test, c cluster.Cluster) { }() // Stop the cluster and restart 2/3 of the nodes. - c.Stop(ctx, option.DefaultStopOpts()) + c.Stop(ctx, t.L(), option.DefaultStopOpts()) tBeginDown := timeutil.Now() - c.Start(ctx, startOpts, settings, c.Range(1, 6)) + c.Start(ctx, t.L(), startOpts, settings, c.Range(1, 6)) waitForUnderReplicated := func(count int) { for start := timeutil.Now(); ; time.Sleep(time.Second) { @@ -394,5 +394,5 @@ FROM crdb_internal.kv_store_status waitForReplication(5) // Restart the down nodes to prevent the dead node detector from complaining. - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(7, 9)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(7, 9)) } diff --git a/pkg/cmd/roachtest/tests/alterpk.go b/pkg/cmd/roachtest/tests/alterpk.go index 197874d126f2..e7e4a95a6d1a 100644 --- a/pkg/cmd/roachtest/tests/alterpk.go +++ b/pkg/cmd/roachtest/tests/alterpk.go @@ -34,7 +34,7 @@ func registerAlterPK(r registry.Registry) { c.Put(ctx, t.DeprecatedWorkload(), "./workload", loadNode) t.Status("starting cockroach nodes") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) return roachNodes, loadNode } @@ -76,7 +76,7 @@ func registerAlterPK(r registry.Registry) { time.Sleep(duration / 10) t.Status("beginning primary key change") - db := c.Conn(ctx, roachNodes[0]) + db := c.Conn(ctx, t.L(), roachNodes[0]) defer db.Close() cmd := ` USE bank; @@ -142,7 +142,7 @@ func registerAlterPK(r registry.Registry) { randStmt := alterStmts[rand.Intn(len(alterStmts))] t.Status("Running command: ", randStmt) - db := c.Conn(ctx, roachNodes[0]) + db := c.Conn(ctx, t.L(), roachNodes[0]) defer db.Close() alterCmd := `USE tpcc; %s;` t.Status("beginning primary key change") diff --git a/pkg/cmd/roachtest/tests/asyncpg.go b/pkg/cmd/roachtest/tests/asyncpg.go index c75ad48e1a21..a1bbf2954e75 100644 --- a/pkg/cmd/roachtest/tests/asyncpg.go +++ b/pkg/cmd/roachtest/tests/asyncpg.go @@ -43,14 +43,14 @@ func registerAsyncpg(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/autoupgrade.go b/pkg/cmd/roachtest/tests/autoupgrade.go index b254abcd00a6..5d5bc903ccfd 100644 --- a/pkg/cmd/roachtest/tests/autoupgrade.go +++ b/pkg/cmd/roachtest/tests/autoupgrade.go @@ -41,7 +41,7 @@ func registerAutoUpgrade(r registry.Registry) { t.Fatal(err) } - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) const stageDuration = 30 * time.Second const timeUntilStoreDead = 90 * time.Second @@ -57,7 +57,7 @@ func registerAutoUpgrade(r registry.Registry) { } } - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() if _, err := db.ExecContext(ctx, @@ -78,7 +78,7 @@ func registerAutoUpgrade(r registry.Registry) { return err } t.WorkerStatus("stop") - c.Stop(ctx, option.DefaultStopOpts(), c.Node(node)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(node)) return nil } @@ -121,13 +121,13 @@ func registerAutoUpgrade(r registry.Registry) { // is testing, i.e. the branch we're running on), except the last node. for i := 1; i < nodes; i++ { t.WorkerStatus("upgrading ", i) - if err := c.StopCockroachGracefullyOnNode(ctx, i); err != nil { + if err := c.StopCockroachGracefullyOnNode(ctx, t.L(), i); err != nil { t.Fatal(err) } c.Put(ctx, t.Cockroach(), "./cockroach", c.Node(i)) startOpts := option.DefaultStartOpts() startOpts.RoachtestOpts.DontEncrypt = true - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Node(i)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(i)) if err := sleep(stageDuration); err != nil { t.Fatal(err) } @@ -142,16 +142,16 @@ func registerAutoUpgrade(r registry.Registry) { // Now stop a previously started node and upgrade the last node. // Check cluster version is not upgraded. - if err := c.StopCockroachGracefullyOnNode(ctx, nodes-1); err != nil { + if err := c.StopCockroachGracefullyOnNode(ctx, t.L(), nodes-1); err != nil { t.Fatal(err) } - if err := c.StopCockroachGracefullyOnNode(ctx, nodes); err != nil { + if err := c.StopCockroachGracefullyOnNode(ctx, t.L(), nodes); err != nil { t.Fatal(err) } c.Put(ctx, t.Cockroach(), "./cockroach", c.Node(nodes)) startOpts := option.DefaultStartOpts() startOpts.RoachtestOpts.DontEncrypt = true - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Node(nodes)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(nodes)) if err := sleep(stageDuration); err != nil { t.Fatal(err) } @@ -196,7 +196,7 @@ func registerAutoUpgrade(r registry.Registry) { // Restart the previously stopped node. startOpts = option.DefaultStartOpts() startOpts.RoachtestOpts.DontEncrypt = true - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Node(nodes-1)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(nodes-1)) if err := sleep(stageDuration); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/backup.go b/pkg/cmd/roachtest/tests/backup.go index 75a0273f6cc7..7b565cf9917e 100644 --- a/pkg/cmd/roachtest/tests/backup.go +++ b/pkg/cmd/roachtest/tests/backup.go @@ -76,7 +76,7 @@ func importBankDataSplit( // NB: starting the cluster creates the logs dir as a side effect, // needed below. - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) runImportBankDataSplit(ctx, rows, ranges, t, c) return dest } @@ -125,8 +125,8 @@ func registerBackupNodeShutdown(r registry.Registry) { nodeToShutdown := 3 dest := loadBackupData(ctx, t, c) backupQuery := `BACKUP bank.bank TO 'nodelocal://1/` + dest + `' WITH DETACHED` - startBackup := func(c cluster.Cluster) (jobID string, err error) { - gatewayDB := c.Conn(ctx, gatewayNode) + startBackup := func(c cluster.Cluster, t test.Test) (jobID string, err error) { + gatewayDB := c.Conn(ctx, t.L(), gatewayNode) defer gatewayDB.Close() err = gatewayDB.QueryRowContext(ctx, backupQuery).Scan(&jobID) @@ -145,8 +145,8 @@ func registerBackupNodeShutdown(r registry.Registry) { nodeToShutdown := 2 dest := loadBackupData(ctx, t, c) backupQuery := `BACKUP bank.bank TO 'nodelocal://1/` + dest + `' WITH DETACHED` - startBackup := func(c cluster.Cluster) (jobID string, err error) { - gatewayDB := c.Conn(ctx, gatewayNode) + startBackup := func(c cluster.Cluster, t test.Test) (jobID string, err error) { + gatewayDB := c.Conn(ctx, t.L(), gatewayNode) defer gatewayDB.Close() err = gatewayDB.QueryRowContext(ctx, backupQuery).Scan(&jobID) @@ -188,7 +188,7 @@ func registerBackupMixedVersion(r registry.Registry) { backupQuery := fmt.Sprintf("BACKUP bank.bank TO 'nodelocal://%d/%s' %s", nodeID, destinationName(c), backupOpts) - gatewayDB := c.Conn(ctx, nodeID) + gatewayDB := c.Conn(ctx, t.L(), nodeID) defer gatewayDB.Close() t.Status("Running: ", backupQuery) _, err := gatewayDB.ExecContext(ctx, backupQuery) @@ -297,7 +297,7 @@ func registerBackup(r registry.Registry) { } dest := importBankData(ctx, rows, t, c) - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) m := c.NewMonitor(ctx) m.Go(func(ctx context.Context) error { _, err := conn.ExecContext(ctx, ` @@ -409,8 +409,8 @@ func registerBackup(r registry.Registry) { c.EncryptAtRandom(true) c.Put(ctx, t.Cockroach(), "./cockroach") c.Put(ctx, t.DeprecatedWorkload(), "./workload") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) - conn := c.Conn(ctx, 1) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) + conn := c.Conn(ctx, t.L(), 1) duration := 5 * time.Minute if c.IsLocal() { diff --git a/pkg/cmd/roachtest/tests/build_info.go b/pkg/cmd/roachtest/tests/build_info.go index e90918cb076b..386622158572 100644 --- a/pkg/cmd/roachtest/tests/build_info.go +++ b/pkg/cmd/roachtest/tests/build_info.go @@ -26,10 +26,10 @@ import ( // RunBuildInfo is a test that sanity checks the build info. func RunBuildInfo(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) var details serverpb.DetailsResponse - adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, c.Node(1)) + adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.Node(1)) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/canary.go b/pkg/cmd/roachtest/tests/canary.go index 841df011ace8..79ffc2709e43 100644 --- a/pkg/cmd/roachtest/tests/canary.go +++ b/pkg/cmd/roachtest/tests/canary.go @@ -22,10 +22,10 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/util/retry" "github.com/cockroachdb/errors" ) @@ -68,8 +68,10 @@ func (b blocklistsForVersion) getLists(version string) (string, blocklist, strin return "", nil, "", nil } -func fetchCockroachVersion(ctx context.Context, c cluster.Cluster, nodeIndex int) (string, error) { - db, err := c.ConnE(ctx, nodeIndex) +func fetchCockroachVersion( + ctx context.Context, l *logger.Logger, c cluster.Cluster, nodeIndex int, +) (string, error) { + db, err := c.ConnE(ctx, l, nodeIndex) if err != nil { return "", err } diff --git a/pkg/cmd/roachtest/tests/cancel.go b/pkg/cmd/roachtest/tests/cancel.go index 7dd550d989df..b07ee3ee121b 100644 --- a/pkg/cmd/roachtest/tests/cancel.go +++ b/pkg/cmd/roachtest/tests/cancel.go @@ -44,7 +44,7 @@ import ( func registerCancel(r registry.Registry) { runCancel := func(ctx context.Context, t test.Test, c cluster.Cluster, tpchQueriesToRun []int, useDistsql bool) { c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) m := c.NewMonitor(ctx, c.All()) m.Go(func(ctx context.Context) error { @@ -53,7 +53,7 @@ func registerCancel(r registry.Registry) { t.Fatal(err) } - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) defer conn.Close() queryPrefix := "USE tpch; " diff --git a/pkg/cmd/roachtest/tests/cdc.go b/pkg/cmd/roachtest/tests/cdc.go index 65a863131c44..9676ffc9b780 100644 --- a/pkg/cmd/roachtest/tests/cdc.go +++ b/pkg/cmd/roachtest/tests/cdc.go @@ -36,13 +36,13 @@ import ( "github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/cdctest" "github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeedbase" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" @@ -111,9 +111,9 @@ func cdcBasicTest(ctx context.Context, t test.Test, c cluster.Cluster, args cdcT kafkaNode := c.Node(c.Spec().NodeCount) c.Put(ctx, t.Cockroach(), "./cockroach") c.Put(ctx, t.DeprecatedWorkload(), "./workload", workloadNode) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer stopFeeds(db) tdb := sqlutils.MakeSQLRunner(db) cdcClusterSettings(t, tdb) @@ -313,7 +313,7 @@ func runCDCBank(ctx context.Context, t test.Test, c cluster.Cluster) { crdbNodes, workloadNode, kafkaNode := c.Range(1, c.Spec().NodeCount-1), c.Node(c.Spec().NodeCount), c.Node(c.Spec().NodeCount) c.Put(ctx, t.Cockroach(), "./cockroach", crdbNodes) c.Put(ctx, t.DeprecatedWorkload(), "./workload", workloadNode) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) kafka := kafkaManager{ t: t, c: c, @@ -336,7 +336,7 @@ func runCDCBank(ctx context.Context, t test.Test, c cluster.Cluster) { } c.Run(ctx, workloadNode, `./workload init bank {pgurl:1}`) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer stopFeeds(db) tdb := sqlutils.MakeSQLRunner(db) @@ -481,7 +481,7 @@ func runCDCSchemaRegistry(ctx context.Context, t test.Test, c cluster.Cluster) { crdbNodes, kafkaNode := c.Node(1), c.Node(1) c.Put(ctx, t.Cockroach(), "./cockroach", crdbNodes) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) kafka := kafkaManager{ t: t, c: c, @@ -491,7 +491,7 @@ func runCDCSchemaRegistry(ctx context.Context, t test.Test, c cluster.Cluster) { kafka.start(ctx) defer kafka.stop(ctx) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer stopFeeds(db) cdcClusterSettings(t, sqlutils.MakeSQLRunner(db)) @@ -620,7 +620,7 @@ func runCDCKafkaAuth(ctx context.Context, t test.Test, c cluster.Cluster) { crdbNodes, kafkaNode := c.Range(1, lastCrdbNode), c.Node(c.Spec().NodeCount) c.Put(ctx, t.Cockroach(), "./cockroach", crdbNodes) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) kafka := kafkaManager{ t: t, @@ -633,7 +633,7 @@ func runCDCKafkaAuth(ctx context.Context, t test.Test, c cluster.Cluster) { kafka.addSCRAMUsers(ctx) defer kafka.stop(ctx) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer stopFeeds(db) tdb := sqlutils.MakeSQLRunner(db) @@ -1280,7 +1280,7 @@ func (k kafkaManager) installJRE(ctx context.Context) error { func (k kafkaManager) configureAuth(ctx context.Context) *testCerts { k.t.Status("generating TLS certificates") - ips, err := k.c.InternalIP(ctx, k.nodes) + ips, err := k.c.InternalIP(ctx, k.t.L(), k.nodes) if err != nil { k.t.Fatal(err) } @@ -1476,7 +1476,7 @@ func (k kafkaManager) chaosLoop( } func (k kafkaManager) sinkURL(ctx context.Context) string { - ips, err := k.c.InternalIP(ctx, k.nodes) + ips, err := k.c.InternalIP(ctx, k.t.L(), k.nodes) if err != nil { k.t.Fatal(err) } @@ -1484,7 +1484,7 @@ func (k kafkaManager) sinkURL(ctx context.Context) string { } func (k kafkaManager) sinkURLTLS(ctx context.Context) string { - ips, err := k.c.InternalIP(ctx, k.nodes) + ips, err := k.c.InternalIP(ctx, k.t.L(), k.nodes) if err != nil { k.t.Fatal(err) } @@ -1492,7 +1492,7 @@ func (k kafkaManager) sinkURLTLS(ctx context.Context) string { } func (k kafkaManager) sinkURLSASL(ctx context.Context) string { - ips, err := k.c.InternalIP(ctx, k.nodes) + ips, err := k.c.InternalIP(ctx, k.t.L(), k.nodes) if err != nil { k.t.Fatal(err) } @@ -1500,7 +1500,7 @@ func (k kafkaManager) sinkURLSASL(ctx context.Context) string { } func (k kafkaManager) consumerURL(ctx context.Context) string { - ips, err := k.c.ExternalIP(ctx, k.nodes) + ips, err := k.c.ExternalIP(ctx, k.t.L(), k.nodes) if err != nil { k.t.Fatal(err) } @@ -1508,7 +1508,7 @@ func (k kafkaManager) consumerURL(ctx context.Context) string { } func (k kafkaManager) schemaRegistryURL(ctx context.Context) string { - ips, err := k.c.InternalIP(ctx, k.nodes) + ips, err := k.c.InternalIP(ctx, k.t.L(), k.nodes) if err != nil { k.t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/chaos.go b/pkg/cmd/roachtest/tests/chaos.go index 5d558fe6ffa6..619f5ff1b122 100644 --- a/pkg/cmd/roachtest/tests/chaos.go +++ b/pkg/cmd/roachtest/tests/chaos.go @@ -139,14 +139,14 @@ func (ch *Chaos) Runner( stopOpts := option.DefaultStopOpts() stopOpts.RoachprodOpts.Sig = 15 stopOpts.RoachtestOpts.Worker = true - if err := c.StopE(ctx, stopOpts, target); err != nil { + if err := c.StopE(ctx, l, stopOpts, target); err != nil { return errors.Wrapf(err, "could not stop node %s", target) } } else { l.Printf("killing %v\n", target) stopOpts := option.DefaultStopOpts() stopOpts.RoachtestOpts.Worker = true - if err := c.StopE(ctx, stopOpts, target); err != nil { + if err := c.StopE(ctx, l, stopOpts, target); err != nil { return errors.Wrapf(err, "could not stop node %s", target) } } @@ -159,7 +159,7 @@ func (ch *Chaos) Runner( l.Printf("restarting %v (chaos is done)\n", target) startOpts := option.DefaultStartOpts() startOpts.RoachtestOpts.Worker = true - if err := c.StartE(ctx, startOpts, install.MakeClusterSettings(), target); err != nil { + if err := c.StartE(ctx, l, startOpts, install.MakeClusterSettings(), target); err != nil { return errors.Wrapf(err, "could not restart node %s", target) } return nil @@ -173,7 +173,7 @@ func (ch *Chaos) Runner( defer cancel() startOpts := option.DefaultStartOpts() startOpts.RoachtestOpts.Worker = true - if err := c.StartE(tCtx, startOpts, install.MakeClusterSettings(), target); err != nil { + if err := c.StartE(tCtx, l, startOpts, install.MakeClusterSettings(), target); err != nil { return errors.Wrapf(err, "could not restart node %s", target) } return ctx.Err() @@ -184,7 +184,7 @@ func (ch *Chaos) Runner( ch.sendEvent(ChaosEventTypePreStartup, target) startOpts := option.DefaultStartOpts() startOpts.RoachtestOpts.Worker = true - if err := c.StartE(ctx, startOpts, install.MakeClusterSettings(), target); err != nil { + if err := c.StartE(ctx, l, startOpts, install.MakeClusterSettings(), target); err != nil { return errors.Wrapf(err, "could not restart node %s", target) } ch.sendEvent(ChaosEventTypeStartupComplete, target) diff --git a/pkg/cmd/roachtest/tests/clearrange.go b/pkg/cmd/roachtest/tests/clearrange.go index f54de186e644..81d967f25384 100644 --- a/pkg/cmd/roachtest/tests/clearrange.go +++ b/pkg/cmd/roachtest/tests/clearrange.go @@ -64,14 +64,14 @@ func runClearRange(ctx context.Context, t test.Test, c cluster.Cluster, aggressi c.Put(ctx, t.Cockroach(), "./cockroach") t.Status("restoring fixture") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) // NB: on a 10 node cluster, this should take well below 3h. tBegin := timeutil.Now() c.Run(ctx, c.Node(1), "./cockroach", "workload", "fixtures", "import", "bank", "--payload-bytes=10240", "--ranges=10", "--rows=65104166", "--seed=4", "--db=bigbank") t.L().Printf("import took %.2fs", timeutil.Since(tBegin).Seconds()) - c.Stop(ctx, option.DefaultStopOpts()) + c.Stop(ctx, t.L(), option.DefaultStopOpts()) t.Status() settings := install.MakeClusterSettings() @@ -83,7 +83,7 @@ func runClearRange(ctx context.Context, t test.Test, c cluster.Cluster, aggressi settings.Env = append(settings.Env, []string{"COCKROACH_CONSISTENCY_AGGRESSIVE=true", "COCKROACH_ENFORCE_CONSISTENT_STATS=true"}...) } - c.Start(ctx, option.DefaultStartOpts(), settings) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings) // Also restore a much smaller table. We'll use it to run queries against // the cluster after having dropped the large table above, verifying that @@ -92,7 +92,7 @@ func runClearRange(ctx context.Context, t test.Test, c cluster.Cluster, aggressi defer t.WorkerStatus() if t.BuildVersion().AtLeast(version.MustParse("v19.2.0")) { - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) if _, err := conn.ExecContext(ctx, `SET CLUSTER SETTING kv.bulk_io_write.concurrent_addsstable_requests = 8`); err != nil { t.Fatal(err) } @@ -111,7 +111,7 @@ func runClearRange(ctx context.Context, t test.Test, c cluster.Cluster, aggressi // Set up a convenience function that we can call to learn the number of // ranges for the bigbank.bank table (even after it's been dropped). numBankRanges := func() func() int { - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) defer conn.Close() var startHex string @@ -121,7 +121,7 @@ func runClearRange(ctx context.Context, t test.Test, c cluster.Cluster, aggressi t.Fatal(err) } return func() int { - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) defer conn.Close() var n int if err := conn.QueryRow( @@ -140,7 +140,7 @@ func runClearRange(ctx context.Context, t test.Test, c cluster.Cluster, aggressi return nil }) m.Go(func(ctx context.Context) error { - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) defer conn.Close() if _, err := conn.ExecContext(ctx, `SET CLUSTER SETTING kv.range_merge.queue_enabled = true`); err != nil { diff --git a/pkg/cmd/roachtest/tests/cli.go b/pkg/cmd/roachtest/tests/cli.go index e8b487d20703..0b2baaba29a6 100644 --- a/pkg/cmd/roachtest/tests/cli.go +++ b/pkg/cmd/roachtest/tests/cli.go @@ -25,9 +25,9 @@ import ( func runCLINodeStatus(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, 3)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, 3)) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() WaitFor3XReplication(t, db) @@ -91,7 +91,7 @@ func runCLINodeStatus(ctx context.Context, t test.Test, c cluster.Cluster) { } // Kill node 2 and wait for it to be marked as !is_available and !is_live. - c.Stop(ctx, option.DefaultStopOpts(), c.Node(2)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(2)) waitUntil([]string{ "is_available is_live", "true true", @@ -104,7 +104,7 @@ func runCLINodeStatus(ctx context.Context, t test.Test, c cluster.Cluster) { // longer write to the liveness range due to lack of quorum. This test is // verifying that "node status" still returns info in this situation since // it only accesses gossip info. - c.Stop(ctx, option.DefaultStopOpts(), c.Node(3)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(3)) waitUntil([]string{ "is_available is_live", "false true", @@ -114,8 +114,8 @@ func runCLINodeStatus(ctx context.Context, t test.Test, c cluster.Cluster) { // Stop the cluster and restart only 2 of the nodes. Verify that three nodes // show up in the node status output. - c.Stop(ctx, option.DefaultStopOpts(), c.Range(1, 2)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, 2)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Range(1, 2)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, 2)) waitUntil([]string{ "is_available is_live", @@ -125,5 +125,5 @@ func runCLINodeStatus(ctx context.Context, t test.Test, c cluster.Cluster) { }) // Start node again to satisfy roachtest. - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(3)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(3)) } diff --git a/pkg/cmd/roachtest/tests/clock_jump_crash.go b/pkg/cmd/roachtest/tests/clock_jump_crash.go index 703cc2f9f885..7c36bb374fbd 100644 --- a/pkg/cmd/roachtest/tests/clock_jump_crash.go +++ b/pkg/cmd/roachtest/tests/clock_jump_crash.go @@ -40,9 +40,9 @@ func runClockJump(ctx context.Context, t test.Test, c cluster.Cluster, tc clockJ c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) } c.Wipe(ctx) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) - db := c.Conn(ctx, c.Spec().NodeCount) + db := c.Conn(ctx, t.L(), c.Spec().NodeCount) defer db.Close() if _, err := db.Exec( fmt.Sprintf( @@ -71,7 +71,7 @@ func runClockJump(ctx context.Context, t test.Test, c cluster.Cluster, tc clockJ // restarting it if not. time.Sleep(3 * time.Second) if !isAlive(db, t.L()) { - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(1)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(1)) } }() defer offsetInjector.recover(ctx, c.Spec().NodeCount) diff --git a/pkg/cmd/roachtest/tests/clock_monotonic.go b/pkg/cmd/roachtest/tests/clock_monotonic.go index 849f53371bcc..fea951e93db1 100644 --- a/pkg/cmd/roachtest/tests/clock_monotonic.go +++ b/pkg/cmd/roachtest/tests/clock_monotonic.go @@ -42,9 +42,9 @@ func runClockMonotonicity( c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) } c.Wipe(ctx) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) - db := c.Conn(ctx, c.Spec().NodeCount) + db := c.Conn(ctx, t.L(), c.Spec().NodeCount) defer db.Close() if _, err := db.Exec( fmt.Sprintf(`SET CLUSTER SETTING server.clock.persist_upper_bound_interval = '%v'`, @@ -71,12 +71,12 @@ func runClockMonotonicity( } // Stop cockroach node before recovering from clock offset as this clock // jump can crash the node. - c.Stop(ctx, option.DefaultStopOpts(), c.Node(c.Spec().NodeCount)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(c.Spec().NodeCount)) t.L().Printf("recovering from injected clock offset") offsetInjector.recover(ctx, c.Spec().NodeCount) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(c.Spec().NodeCount)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(c.Spec().NodeCount)) if !isAlive(db, t.L()) { t.Fatal("Node unexpectedly crashed") } @@ -84,11 +84,11 @@ func runClockMonotonicity( // Inject a clock offset after stopping a node t.Status("stopping cockroach") - c.Stop(ctx, option.DefaultStopOpts(), c.Node(c.Spec().NodeCount)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(c.Spec().NodeCount)) t.Status("injecting offset") offsetInjector.offset(ctx, c.Spec().NodeCount, tc.offset) t.Status("starting cockroach post offset") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(c.Spec().NodeCount)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(c.Spec().NodeCount)) if !isAlive(db, t.L()) { t.Fatal("Node unexpectedly crashed") diff --git a/pkg/cmd/roachtest/tests/clock_util.go b/pkg/cmd/roachtest/tests/clock_util.go index 14cc859875d7..9d5459fe11c4 100644 --- a/pkg/cmd/roachtest/tests/clock_util.go +++ b/pkg/cmd/roachtest/tests/clock_util.go @@ -17,8 +17,8 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" ) // isAlive returns whether the node queried by db is alive. @@ -59,10 +59,10 @@ func (oi *offsetInjector) deploy(ctx context.Context) error { return nil } - if err := oi.c.Install(ctx, oi.c.All(), "ntp"); err != nil { + if err := oi.c.Install(ctx, oi.t.L(), oi.c.All(), "ntp"); err != nil { return err } - if err := oi.c.Install(ctx, oi.c.All(), "gcc"); err != nil { + if err := oi.c.Install(ctx, oi.t.L(), oi.c.All(), "gcc"); err != nil { return err } if err := oi.c.RunE(ctx, oi.c.All(), "sudo", "service", "ntp", "stop"); err != nil { diff --git a/pkg/cmd/roachtest/tests/cluster_init.go b/pkg/cmd/roachtest/tests/cluster_init.go index 74e10a9dbcfe..dc2b1e39dfaf 100644 --- a/pkg/cmd/roachtest/tests/cluster_init.go +++ b/pkg/cmd/roachtest/tests/cluster_init.go @@ -33,7 +33,7 @@ func runClusterInit(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") t.L().Printf("retrieving VM addresses") - addrs, err := c.InternalAddr(ctx, c.All()) + addrs, err := c.InternalAddr(ctx, t.L(), c.All()) if err != nil { t.Fatal(err) } @@ -63,10 +63,10 @@ func runClusterInit(ctx context.Context, t test.Test, c cluster.Cluster) { // that all nodes can discover the init'ed node (transitively) // via their join flags. startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--join="+strings.Join(addrs, ",")) - c.Start(ctx, startOpts, install.MakeClusterSettings()) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings()) urlMap := make(map[int]string) - adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, c.All()) + adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.All()) if err != nil { t.Fatal(err) } @@ -91,7 +91,7 @@ func runClusterInit(ctx context.Context, t test.Test, c cluster.Cluster) { var dbs []*gosql.DB for i := 1; i <= c.Spec().NodeCount; i++ { - db := c.Conn(ctx, i) + db := c.Conn(ctx, t.L(), i) defer db.Close() dbs = append(dbs, db) } diff --git a/pkg/cmd/roachtest/tests/connection_latency.go b/pkg/cmd/roachtest/tests/connection_latency.go index c8c293e71046..09f4b89253a4 100644 --- a/pkg/cmd/roachtest/tests/connection_latency.go +++ b/pkg/cmd/roachtest/tests/connection_latency.go @@ -41,7 +41,7 @@ func runConnectionLatencyTest( require.NoError(t, err) settings := install.MakeClusterSettings(install.SecureOption(true)) - err = c.StartE(ctx, option.DefaultStartOpts(), settings) + err = c.StartE(ctx, t.L(), option.DefaultStartOpts(), settings) require.NoError(t, err) var passwordFlag string @@ -65,7 +65,7 @@ func runConnectionLatencyTest( runWorkload := func(roachNodes, loadNode option.NodeListOption, locality string) { var urlString string var urls []string - externalIps, err := c.ExternalIP(ctx, roachNodes) + externalIps, err := c.ExternalIP(ctx, t.L(), roachNodes) require.NoError(t, err) if password { diff --git a/pkg/cmd/roachtest/tests/copy.go b/pkg/cmd/roachtest/tests/copy.go index 58a197b2d517..7b21fd066406 100644 --- a/pkg/cmd/roachtest/tests/copy.go +++ b/pkg/cmd/roachtest/tests/copy.go @@ -44,11 +44,11 @@ func registerCopy(r registry.Registry) { c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) m := c.NewMonitor(ctx, c.All()) m.Go(func(ctx context.Context) error { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() // Disable load-based splitting so that we can more accurately diff --git a/pkg/cmd/roachtest/tests/decommission.go b/pkg/cmd/roachtest/tests/decommission.go index 0e814f981042..e164bc12bae2 100644 --- a/pkg/cmd/roachtest/tests/decommission.go +++ b/pkg/cmd/roachtest/tests/decommission.go @@ -105,12 +105,12 @@ func runDrainAndDecommission( pinnedNode := 1 c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) for i := 1; i <= nodes; i++ { - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(i)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(i)) } c.Run(ctx, c.Node(pinnedNode), `./cockroach workload init kv --drop --splits 1000`) run := func(stmt string) { - db := c.Conn(ctx, pinnedNode) + db := c.Conn(ctx, t.L(), pinnedNode) defer db.Close() t.Status(stmt) @@ -129,7 +129,7 @@ func runDrainAndDecommission( run(`SET CLUSTER SETTING kv.snapshot_recovery.max_rate='2GiB'`) t.Status("waiting for initial up-replication") - db := c.Conn(ctx, pinnedNode) + db := c.Conn(ctx, t.L(), pinnedNode) defer func() { _ = db.Close() }() @@ -225,12 +225,12 @@ func runDecommission( for i := 1; i <= nodes; i++ { startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, fmt.Sprintf("--attrs=node%d", i)) - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Node(i)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(i)) } c.Run(ctx, c.Node(pinnedNode), `./workload init kv --drop`) waitReplicatedAwayFrom := func(downNodeID int) error { - db := c.Conn(ctx, pinnedNode) + db := c.Conn(ctx, t.L(), pinnedNode) defer func() { _ = db.Close() }() @@ -262,7 +262,7 @@ func runDecommission( } waitUpReplicated := func(targetNode, targetNodeID int) error { - db := c.Conn(ctx, pinnedNode) + db := c.Conn(ctx, t.L(), pinnedNode) defer func() { _ = db.Close() }() @@ -297,7 +297,7 @@ func runDecommission( } run := func(stmt string) { - db := c.Conn(ctx, pinnedNode) + db := c.Conn(ctx, t.L(), pinnedNode) defer db.Close() t.Status(stmt) @@ -319,7 +319,7 @@ func runDecommission( m.Go(func() error { getNodeID := func(node int) (int, error) { - dbNode := c.Conn(ctx, node) + dbNode := c.Conn(ctx, t.L(), node) defer dbNode.Close() var nodeID int @@ -391,16 +391,16 @@ func runDecommission( return err } - db := c.Conn(ctx, pinnedNode) + db := c.Conn(ctx, t.L(), pinnedNode) defer db.Close() - internalAddrs, err := c.InternalAddr(ctx, c.Node(pinnedNode)) + internalAddrs, err := c.InternalAddr(ctx, t.L(), c.Node(pinnedNode)) if err != nil { return err } startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, fmt.Sprintf("--join %s --attrs=node%d", internalAddrs[0], node)) - if err := c.StartE(ctx, startOpts, install.MakeClusterSettings(), c.Node(node)); err != nil { + if err := c.StartE(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(node)); err != nil { return err } } @@ -424,7 +424,7 @@ func runDecommissionRandomized(ctx context.Context, t test.Test, c cluster.Clust c.Put(ctx, t.Cockroach(), "./cockroach") settings := install.MakeClusterSettings() settings.Env = append(settings.Env, "COCKROACH_SCAN_MAX_IDLE_TIME=5ms") - c.Start(ctx, option.DefaultStartOpts(), settings) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings) h := newDecommTestHelper(t, c) @@ -575,7 +575,7 @@ func runDecommissionRandomized(ctx context.Context, t test.Test, c cluster.Clust { runNode := h.getRandNode() t.L().Printf("checking that we're able to create a database (from n%d)\n", runNode) - db := c.Conn(ctx, runNode) + db := c.Conn(ctx, t.L(), runNode) defer db.Close() if _, err := db.Exec(`create database still_working;`); err != nil { @@ -758,14 +758,14 @@ func runDecommissionRandomized(ctx context.Context, t test.Test, c cluster.Clust runNode = h.getRandNode() t.L().Printf("expected to fail: restarting [n%d,n%d] and attempting to recommission through n%d\n", targetNodeA, targetNodeB, runNode) - c.Stop(ctx, option.DefaultStopOpts(), c.Nodes(targetNodeA, targetNodeB)) - c.Start(ctx, option.DefaultStartOpts(), settings, c.Nodes(targetNodeA, targetNodeB)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Nodes(targetNodeA, targetNodeB)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, c.Nodes(targetNodeA, targetNodeB)) if _, err := h.recommission(ctx, c.Nodes(targetNodeA, targetNodeB), runNode); err == nil { t.Fatalf("expected recommission to fail") } // Now stop+wipe them for good to keep the logs simple and the dead node detector happy. - c.Stop(ctx, option.DefaultStopOpts(), c.Nodes(targetNodeA, targetNodeB)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Nodes(targetNodeA, targetNodeB)) c.Wipe(ctx, c.Nodes(targetNodeA, targetNodeB)) } } @@ -789,7 +789,7 @@ func runDecommissionRandomized(ctx context.Context, t test.Test, c cluster.Clust // everything will seem dead to the allocator at all times, so // nothing will ever happen. func() { - db := c.Conn(ctx, h.getRandNode()) + db := c.Conn(ctx, t.L(), h.getRandNode()) defer db.Close() const stmt = "SET CLUSTER SETTING server.time_until_store_dead = '1m15s'" if _, err := db.ExecContext(ctx, stmt); err != nil { @@ -805,7 +805,7 @@ func runDecommissionRandomized(ctx context.Context, t test.Test, c cluster.Clust targetNode := h.getRandNodeOtherThan(firstNodeID) h.blockFromRandNode(targetNode) t.L().Printf("intentionally killing n%d to later decommission it when down\n", targetNode) - c.Stop(ctx, option.DefaultStopOpts(), c.Node(targetNode)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(targetNode)) // Pick a runNode that is still in commission and will // remain so (or it won't be able to contact cluster). @@ -821,7 +821,7 @@ func runDecommissionRandomized(ctx context.Context, t test.Test, c cluster.Clust // Bring targetNode it back up to verify that its replicas still get // removed. - c.Start(ctx, option.DefaultStartOpts(), settings, c.Node(targetNode)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, c.Node(targetNode)) } // Run decommission a second time to wait until the replicas have @@ -888,18 +888,18 @@ func runDecommissionRandomized(ctx context.Context, t test.Test, c cluster.Clust { t.L().Printf("wiping n%d and adding it back to the cluster as a new node\n", targetNode) - c.Stop(ctx, option.DefaultStopOpts(), c.Node(targetNode)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(targetNode)) c.Wipe(ctx, c.Node(targetNode)) joinNode := h.getRandNode() - internalAddrs, err := c.InternalAddr(ctx, c.Node(joinNode)) + internalAddrs, err := c.InternalAddr(ctx, t.L(), c.Node(joinNode)) if err != nil { t.Fatal(err) } joinAddr := internalAddrs[0] startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, fmt.Sprintf("--join %s", joinAddr)) - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Node(targetNode)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(targetNode)) } if err := retry.WithMaxAttempts(ctx, retryOpts, 50, func() error { @@ -934,7 +934,7 @@ func runDecommissionRandomized(ctx context.Context, t test.Test, c cluster.Clust if err := retry.ForDuration(time.Minute, func() error { // Verify the event log has recorded exactly one decommissioned or // recommissioned event for each membership operation. - db := c.Conn(ctx, h.getRandNode()) + db := c.Conn(ctx, t.L(), h.getRandNode()) defer db.Close() rows, err := db.Query(` diff --git a/pkg/cmd/roachtest/tests/disk_full.go b/pkg/cmd/roachtest/tests/disk_full.go index 52ba7d2d0652..358c6b0f0b39 100644 --- a/pkg/cmd/roachtest/tests/disk_full.go +++ b/pkg/cmd/roachtest/tests/disk_full.go @@ -37,7 +37,7 @@ func registerDiskFull(r registry.Registry) { nodes := c.Spec().NodeCount - 1 c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, nodes)) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(nodes+1)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) t.Status("running workload") m := c.NewMonitor(ctx, c.Range(1, nodes)) @@ -66,7 +66,7 @@ func registerDiskFull(r registry.Registry) { // Node 1 should forcibly exit due to a full disk. for isLive := true; isLive; { - db := c.Conn(ctx, 2) + db := c.Conn(ctx, t.L(), 2) err := db.QueryRow(`SELECT is_live FROM crdb_internal.gossip_nodes WHERE node_id = 1;`).Scan(&isLive) if err != nil { t.Fatal(err) @@ -91,7 +91,7 @@ func registerDiskFull(r registry.Registry) { // monitor detects the death, expect it. m.ExpectDeath() - err := c.StartE(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(n)) + err := c.StartE(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(n)) t.L().Printf("starting n%d: error %v", n, err) if err == nil { t.Fatal("node successfully started unexpectedly") @@ -122,7 +122,7 @@ func registerDiskFull(r registry.Registry) { t.L().Printf("removing the emergency ballast on n%d\n", n) m.ResetDeaths() c.Run(ctx, c.Node(n), "rm -f {store-dir}/auxiliary/EMERGENCY_BALLAST") - if err := c.StartE(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(n)); err != nil { + if err := c.StartE(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(n)); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/disk_stall.go b/pkg/cmd/roachtest/tests/disk_stall.go index d688a1b9b097..66c7a96abee4 100644 --- a/pkg/cmd/roachtest/tests/disk_stall.go +++ b/pkg/cmd/roachtest/tests/disk_stall.go @@ -65,7 +65,7 @@ func runDiskStalledDetection( t.Status("setting up charybdefs") - if err := c.Install(ctx, n, "charybdefs"); err != nil { + if err := c.Install(ctx, t.L(), n, "charybdefs"); err != nil { t.Fatal(err) } c.Run(ctx, n, "sudo charybdefs {store-dir}/faulty -oallow_other,modules=subdir,subdir={store-dir}/real") diff --git a/pkg/cmd/roachtest/tests/django.go b/pkg/cmd/roachtest/tests/django.go index f861a6018c66..4b1b89687e5a 100644 --- a/pkg/cmd/roachtest/tests/django.go +++ b/pkg/cmd/roachtest/tests/django.go @@ -42,14 +42,14 @@ func registerDjango(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - err = alterZoneConfigAndClusterSettings(ctx, version, c, node[0]) + err = alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/drop.go b/pkg/cmd/roachtest/tests/drop.go index 7896d151af37..fdb37aad7267 100644 --- a/pkg/cmd/roachtest/tests/drop.go +++ b/pkg/cmd/roachtest/tests/drop.go @@ -38,7 +38,7 @@ func registerDrop(r registry.Registry) { c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Range(1, nodes)) settings := install.MakeClusterSettings() settings.Env = append(settings.Env, "COCKROACH_MEMPROF_INTERVAL=15s") - c.Start(ctx, option.DefaultStartOpts(), settings, c.Range(1, nodes)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, c.Range(1, nodes)) m := c.NewMonitor(ctx, c.Range(1, nodes)) m.Go(func(ctx context.Context) error { @@ -49,7 +49,7 @@ func registerDrop(r registry.Registry) { // Otherwise the ALTER TABLE query below might fail to find the // tpcc.order_line table that we just imported (!) due to what seems to // be a problem with table descriptor leases (#24374). - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() run := func(maybeExperimental bool, stmtStr string, args ...interface{}) { diff --git a/pkg/cmd/roachtest/tests/drt.go b/pkg/cmd/roachtest/tests/drt.go index 168678b0ca39..37c1812bfb74 100644 --- a/pkg/cmd/roachtest/tests/drt.go +++ b/pkg/cmd/roachtest/tests/drt.go @@ -17,8 +17,8 @@ import ( "fmt" "time" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/prometheus" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/errors" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" diff --git a/pkg/cmd/roachtest/tests/drt_test.go b/pkg/cmd/roachtest/tests/drt_test.go index e90180426b99..01a52bbcf8e7 100644 --- a/pkg/cmd/roachtest/tests/drt_test.go +++ b/pkg/cmd/roachtest/tests/drt_test.go @@ -16,9 +16,9 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/prometheus" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" gomock "github.com/golang/mock/gomock" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" diff --git a/pkg/cmd/roachtest/tests/encryption.go b/pkg/cmd/roachtest/tests/encryption.go index 8690db249b1b..84206593d057 100644 --- a/pkg/cmd/roachtest/tests/encryption.go +++ b/pkg/cmd/roachtest/tests/encryption.go @@ -31,10 +31,10 @@ func registerEncryption(r registry.Registry) { c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, nodes)) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.EncryptedStores = true - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) // Check that /_status/stores/local endpoint has encryption status. - adminAddrs, err := c.InternalAdminUIAddr(ctx, c.Range(1, nodes)) + adminAddrs, err := c.InternalAdminUIAddr(ctx, t.L(), c.Range(1, nodes)) if err != nil { t.Fatal(err) } @@ -45,13 +45,13 @@ func registerEncryption(r registry.Registry) { } for i := 1; i <= nodes; i++ { - if err := c.StopCockroachGracefullyOnNode(ctx, i); err != nil { + if err := c.StopCockroachGracefullyOnNode(ctx, t.L(), i); err != nil { t.Fatal(err) } } // Restart node with encryption turned on to verify old key works. - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) testCLIGenKey := func(size int) error { // Generate encryption store key through `./cockroach gen encryption-key -s=size aes-size.key`. diff --git a/pkg/cmd/roachtest/tests/engine_switch.go b/pkg/cmd/roachtest/tests/engine_switch.go index 5667d0d5208c..c4f2c7e8a0c8 100644 --- a/pkg/cmd/roachtest/tests/engine_switch.go +++ b/pkg/cmd/roachtest/tests/engine_switch.go @@ -38,7 +38,7 @@ func registerEngineSwitch(r registry.Registry) { pebbleStartOpts := option.DefaultStartOpts() pebbleStartOpts.RoachprodOpts.ExtraArgs = append(pebbleStartOpts.RoachprodOpts.ExtraArgs, "--storage-engine=pebble") - c.Start(ctx, rockdbStartOpts, install.MakeClusterSettings(), roachNodes) + c.Start(ctx, t.L(), rockdbStartOpts, install.MakeClusterSettings(), roachNodes) stageDuration := 1 * time.Minute if c.IsLocal() { t.L().Printf("local mode: speeding up test\n") @@ -87,7 +87,7 @@ func registerEngineSwitch(r registry.Registry) { // Make sure everyone is still running. for i := 1; i <= len(roachNodes); i++ { t.WorkerStatus("checking ", i) - db := c.Conn(ctx, i) + db := c.Conn(ctx, t.L(), i) defer db.Close() rows, err := db.Query(`SHOW DATABASES`) if err != nil { @@ -113,10 +113,10 @@ func registerEngineSwitch(r registry.Registry) { m.ExpectDeath() if rng.Intn(2) == 0 { l.Printf("stopping node gracefully %d\n", node) - return c.StopCockroachGracefullyOnNode(ctx, node) + return c.StopCockroachGracefullyOnNode(ctx, t.L(), node) } l.Printf("stopping node %d\n", node) - c.Stop(ctx, option.DefaultStopOpts(), c.Node(node)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(node)) return nil } @@ -133,7 +133,7 @@ func registerEngineSwitch(r registry.Registry) { if err := stop(i + 1); err != nil { return err } - c.Start(ctx, opts, install.MakeClusterSettings(), c.Node(i+1)) + c.Start(ctx, t.L(), opts, install.MakeClusterSettings(), c.Node(i+1)) } return sleepAndCheck() }) diff --git a/pkg/cmd/roachtest/tests/event_log.go b/pkg/cmd/roachtest/tests/event_log.go index 3c594d99c772..04fac03e7abe 100644 --- a/pkg/cmd/roachtest/tests/event_log.go +++ b/pkg/cmd/roachtest/tests/event_log.go @@ -31,11 +31,11 @@ func runEventLog(ctx context.Context, t test.Test, c cluster.Cluster) { } c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) // Verify that "node joined" and "node restart" events are recorded whenever // a node starts and contacts the cluster. - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() WaitFor3XReplication(t, db) @@ -86,8 +86,8 @@ func runEventLog(ctx context.Context, t test.Test, c cluster.Cluster) { } // Stop and Start Node 3, and verify the node restart message. - c.Stop(ctx, option.DefaultStopOpts(), c.Node(3)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(3)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(3)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(3)) err = retry.ForDuration(10*time.Second, func() error { // Query all node restart events. There should only be one. diff --git a/pkg/cmd/roachtest/tests/flowable.go b/pkg/cmd/roachtest/tests/flowable.go index 41c732568a06..81f63f904dc5 100644 --- a/pkg/cmd/roachtest/tests/flowable.go +++ b/pkg/cmd/roachtest/tests/flowable.go @@ -37,7 +37,7 @@ func registerFlowable(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) t.Status("cloning flowable and installing prerequisites") latestTag, err := repeatGetLatestTag( diff --git a/pkg/cmd/roachtest/tests/follower_reads.go b/pkg/cmd/roachtest/tests/follower_reads.go index 94ea5fbba4f0..74c069e3a563 100644 --- a/pkg/cmd/roachtest/tests/follower_reads.go +++ b/pkg/cmd/roachtest/tests/follower_reads.go @@ -62,7 +62,7 @@ func registerFollowerReads(r registry.Registry) { ), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) topology := topologySpec{ multiRegion: true, locality: locality, @@ -169,7 +169,7 @@ func runFollowerReadsTest( ) { var conns []*gosql.DB for i := 0; i < c.Spec().NodeCount; i++ { - conns = append(conns, c.Conn(ctx, i+1)) + conns = append(conns, c.Conn(ctx, t.L(), i+1)) defer conns[i].Close() } db := conns[0] @@ -266,7 +266,7 @@ func runFollowerReadsTest( // the delta and protect from follower reads which might have happened due to // system queries. time.Sleep(10 * time.Second) // wait a bit, otherwise sometimes I get a 404 below. - followerReadsBefore, err := getFollowerReadCounts(ctx, c) + followerReadsBefore, err := getFollowerReadCounts(ctx, t, c) if err != nil { t.Fatalf("failed to get follower read counts: %v", err) } @@ -293,7 +293,7 @@ func runFollowerReadsTest( // Verify that the follower read count increments on at least two nodes - // which we expect to be in the non-primary regions. expNodesToSeeFollowerReads := 2 - followerReadsAfter, err := getFollowerReadCounts(ctx, c) + followerReadsAfter, err := getFollowerReadCounts(ctx, t, c) if err != nil { t.Fatalf("failed to get follower read counts: %v", err) } @@ -314,7 +314,7 @@ func runFollowerReadsTest( if topology.deadPrimaryRegion && i <= 3 { stopOpts := option.DefaultStopOpts() stopOpts.RoachprodOpts.Sig = 9 - c.Stop(ctx, stopOpts, c.Node(i)) + c.Stop(ctx, t.L(), stopOpts, c.Node(i)) deadNodes[i] = struct{}{} } else { liveNodes[i] = struct{}{} @@ -385,7 +385,7 @@ func runFollowerReadsTest( // Restart dead nodes, if necessary. for i := range deadNodes { - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(i)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(i)) } } @@ -394,7 +394,7 @@ func runFollowerReadsTest( func initFollowerReadsDB( ctx context.Context, t test.Test, c cluster.Cluster, topology topologySpec, ) (data map[int]int64) { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) // Disable load based splitting and range merging because splits and merges // interfere with follower reads. This test's workload regularly triggers load // based splitting in the first phase creating small ranges which later @@ -619,7 +619,7 @@ func verifySQLLatency( adminNode = i break } - adminURLs, err := c.ExternalAdminUIAddr(ctx, c.Node(adminNode)) + adminURLs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.Node(adminNode)) if err != nil { t.Fatal(err) } @@ -686,7 +686,7 @@ func verifyHighFollowerReadRatios( adminNode = i break } - adminURLs, err := c.ExternalAdminUIAddr(ctx, c.Node(adminNode)) + adminURLs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.Node(adminNode)) require.NoError(t, err) url := "http://" + adminURLs[0] + "/ts/query" request := tspb.TimeSeriesQueryRequest{ @@ -788,11 +788,11 @@ const followerReadsMetric = "follower_reads_success_count" // getFollowerReadCounts returns a slice from node to follower read count // according to the metric. -func getFollowerReadCounts(ctx context.Context, c cluster.Cluster) ([]int, error) { +func getFollowerReadCounts(ctx context.Context, t test.Test, c cluster.Cluster) ([]int, error) { followerReadCounts := make([]int, c.Spec().NodeCount) getFollowerReadCount := func(ctx context.Context, node int) func() error { return func() error { - adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, c.Node(node)) + adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.Node(node)) if err != nil { return err } @@ -873,7 +873,7 @@ func runFollowerReadsMixedVersionSingleRegionTest( // Start the cluster at the old version. settings := install.MakeClusterSettings() settings.Binary = uploadVersion(ctx, t, c, c.All(), predecessorVersion) - c.Start(ctx, option.DefaultStartOpts(), settings, c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, c.All()) topology := topologySpec{multiRegion: false} data := initFollowerReadsDB(ctx, t, c, topology) diff --git a/pkg/cmd/roachtest/tests/gopg.go b/pkg/cmd/roachtest/tests/gopg.go index f295e9eae870..f3075f43153d 100644 --- a/pkg/cmd/roachtest/tests/gopg.go +++ b/pkg/cmd/roachtest/tests/gopg.go @@ -50,13 +50,13 @@ func registerGopg(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/gorm.go b/pkg/cmd/roachtest/tests/gorm.go index c2fe953cb462..1ecc62a61788 100644 --- a/pkg/cmd/roachtest/tests/gorm.go +++ b/pkg/cmd/roachtest/tests/gorm.go @@ -34,12 +34,12 @@ func registerGORM(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/gossip.go b/pkg/cmd/roachtest/tests/gossip.go index 4bf5eda21915..960b3735deef 100644 --- a/pkg/cmd/roachtest/tests/gossip.go +++ b/pkg/cmd/roachtest/tests/gossip.go @@ -29,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/httputil" "github.com/cockroachdb/cockroach/pkg/util/retry" @@ -41,8 +42,8 @@ func registerGossip(r registry.Registry) { c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--vmodule=*=1") - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.All()) - WaitFor3XReplication(t, c.Conn(ctx, 1)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.All()) + WaitFor3XReplication(t, c.Conn(ctx, t.L(), 1)) // TODO(irfansharif): We could also look at gossip_liveness to determine // cluster membership as seen by each gossip module, and ensure each @@ -54,7 +55,7 @@ SELECT string_agg(source_id::TEXT || ':' || target_id::TEXT, ',') FROM (SELECT * FROM crdb_internal.gossip_network ORDER BY source_id, target_id) ` - db := c.Conn(ctx, node) + db := c.Conn(ctx, t.L(), node) defer db.Close() var s gosql.NullString if err := db.QueryRow(query).Scan(&s); err != nil { @@ -149,9 +150,9 @@ SELECT string_agg(source_id::TEXT || ':' || target_id::TEXT, ',') nodes := c.All() for j := 0; j < 10; j++ { deadNode := nodes.RandNode()[0] - c.Stop(ctx, option.DefaultStopOpts(), c.Node(deadNode)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(deadNode)) waitForGossip(deadNode) - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Node(deadNode)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(deadNode)) } } @@ -168,12 +169,12 @@ SELECT string_agg(source_id::TEXT || ':' || target_id::TEXT, ',') type gossipUtil struct { waitTime time.Duration urlMap map[int]string - conn func(ctx context.Context, i int) *gosql.DB + conn func(ctx context.Context, l *logger.Logger, i int) *gosql.DB } func newGossipUtil(ctx context.Context, t test.Test, c cluster.Cluster) *gossipUtil { urlMap := make(map[int]string) - adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, c.All()) + adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.All()) if err != nil { t.Fatal(err) } @@ -257,7 +258,7 @@ func (g *gossipUtil) checkConnectedAndFunctional( } for i := 1; i <= c.Spec().NodeCount; i++ { - db := g.conn(ctx, i) + db := g.conn(ctx, t.L(), i) defer db.Close() if i == 1 { if _, err := db.Exec("CREATE DATABASE IF NOT EXISTS test"); err != nil { @@ -291,7 +292,7 @@ func (g *gossipUtil) checkConnectedAndFunctional( func runGossipPeerings(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) // Repeatedly restart a random node and verify that all of the nodes are // seeing the gossiped values. @@ -314,8 +315,8 @@ func runGossipPeerings(ctx context.Context, t test.Test, c cluster.Cluster) { // Restart a random node. node := c.All().RandNode() t.L().Printf("%d: restarting node %d\n", i, node[0]) - c.Stop(ctx, option.DefaultStopOpts(), node) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), node) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), node) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), node) // Sleep a bit to avoid hitting: // https://github.com/cockroachdb/cockroach/issues/48005 time.Sleep(3 * time.Second) @@ -326,7 +327,7 @@ func runGossipRestart(ctx context.Context, t test.Test, c cluster.Cluster) { t.Skip("skipping flaky acceptance/gossip/restart", "https://github.com/cockroachdb/cockroach/issues/48423") c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) // Repeatedly stop and restart a cluster and verify that we can perform basic // operations. This is stressing the gossiping of the first range descriptor @@ -341,10 +342,10 @@ func runGossipRestart(ctx context.Context, t test.Test, c cluster.Cluster) { t.L().Printf("%d: OK\n", i) t.L().Printf("%d: killing all nodes\n", i) - c.Stop(ctx, option.DefaultStopOpts()) + c.Stop(ctx, t.L(), option.DefaultStopOpts()) t.L().Printf("%d: restarting all nodes\n", i) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) } } @@ -356,9 +357,9 @@ func runGossipRestartNodeOne(ctx context.Context, t test.Test, c cluster.Cluster startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.EncryptedStores = false - c.Start(ctx, startOpts, settings) + c.Start(ctx, t.L(), startOpts, settings) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() run := func(stmtStr string) { @@ -445,7 +446,7 @@ SELECT count(replicas) } t.L().Printf("killing all nodes\n") - c.Stop(ctx, option.DefaultStopOpts()) + c.Stop(ctx, t.L(), option.DefaultStopOpts()) // Restart node 1, but have it listen on a different port for internal // connections. This will require node 1 to reach out to the other nodes in @@ -465,18 +466,18 @@ SELECT count(replicas) // incoming gossip info in order to determine where range 1 is. settings = install.MakeClusterSettings() settings.Env = append(settings.Env, "COCKROACH_SCAN_MAX_IDLE_TIME=5ms") - c.Start(ctx, startOpts, settings, c.Range(2, c.Spec().NodeCount)) + c.Start(ctx, t.L(), startOpts, settings, c.Range(2, c.Spec().NodeCount)) // We need to override DB connection creation to use the correct port for // node 1. This is more complicated than it should be and a limitation of the // current infrastructure which doesn't know about cockroach nodes started on // non-standard ports. g := newGossipUtil(ctx, t, c) - g.conn = func(ctx context.Context, i int) *gosql.DB { + g.conn = func(ctx context.Context, l *logger.Logger, i int) *gosql.DB { if i != 1 { - return c.Conn(ctx, i) + return c.Conn(ctx, l, i) } - urls, err := c.ExternalPGUrl(ctx, c.Node(1)) + urls, err := c.ExternalPGUrl(ctx, l, c.Node(1)) if err != nil { t.Fatal(err) } @@ -504,14 +505,14 @@ SELECT count(replicas) // Stop our special snowflake process which won't be recognized by the test // harness, and start it again on the regular. - c.Stop(ctx, option.DefaultStopOpts(), c.Node(1)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(1)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(1)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(1)) } func runCheckLocalityIPAddress(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") - externalIP, err := c.ExternalIP(ctx, c.Range(1, c.Spec().NodeCount)) + externalIP, err := c.ExternalIP(ctx, t.L(), c.Range(1, c.Spec().NodeCount)) if err != nil { t.Fatal(err) } @@ -525,13 +526,13 @@ func runCheckLocalityIPAddress(ctx context.Context, t test.Test, c cluster.Clust settings := install.MakeClusterSettings(install.NumRacksOption(1)) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, fmt.Sprintf("--locality-advertise-addr=rack=0@%s", extAddr)) - c.Start(ctx, startOpts, settings, c.Node(i)) + c.Start(ctx, t.L(), startOpts, settings, c.Node(i)) } rowCount := 0 for i := 1; i <= c.Spec().NodeCount; i++ { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() rows, err := db.Query( @@ -554,7 +555,7 @@ func runCheckLocalityIPAddress(ctx context.Context, t test.Test, c cluster.Clust t.Fatal("Expected connect address to contain localhost") } } else { - exps, err := c.ExternalAddr(ctx, c.Node(nodeID)) + exps, err := c.ExternalAddr(ctx, t.L(), c.Node(nodeID)) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/hibernate.go b/pkg/cmd/roachtest/tests/hibernate.go index 77d72b88ed3e..0bd0949a1477 100644 --- a/pkg/cmd/roachtest/tests/hibernate.go +++ b/pkg/cmd/roachtest/tests/hibernate.go @@ -53,7 +53,7 @@ var ( `HIBERNATE_CONNECTION_LEAK_DETECTION=true ./../gradlew test -Pdb=cockroachdb_spatial`, blocklists: hibernateSpatialBlocklists, dbSetupFunc: func(ctx context.Context, t test.Test, c cluster.Cluster) { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() if _, err := db.ExecContext( ctx, @@ -83,18 +83,18 @@ func registerHibernate(r registry.Registry, opt hibernateOptions) { if err := c.PutLibraries(ctx, "./lib"); err != nil { t.Fatal(err) } - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) if opt.dbSetupFunc != nil { opt.dbSetupFunc(ctx, t, c) } - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/hotspotsplits.go b/pkg/cmd/roachtest/tests/hotspotsplits.go index 192c2659a3cf..9604ab5b1925 100644 --- a/pkg/cmd/roachtest/tests/hotspotsplits.go +++ b/pkg/cmd/roachtest/tests/hotspotsplits.go @@ -35,7 +35,7 @@ func registerHotSpotSplits(r registry.Registry) { appNode := c.Node(c.Spec().NodeCount) c.Put(ctx, t.Cockroach(), "./cockroach", roachNodes) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) c.Put(ctx, t.DeprecatedWorkload(), "./workload", appNode) c.Run(ctx, appNode, `./workload init kv --drop {pgurl:1}`) @@ -57,7 +57,7 @@ func registerHotSpotSplits(r registry.Registry) { t.Status("starting checks for range sizes") const sizeLimit = 3 * (1 << 29) // 3*512 MB (512 mb is default size) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() var size = float64(0) diff --git a/pkg/cmd/roachtest/tests/import.go b/pkg/cmd/roachtest/tests/import.go index ccea5cc771f5..c02c889b397b 100644 --- a/pkg/cmd/roachtest/tests/import.go +++ b/pkg/cmd/roachtest/tests/import.go @@ -39,8 +39,8 @@ func readCreateTableFromFixture(fixtureURI string, gatewayDB *gosql.DB) (string, } func registerImportNodeShutdown(r registry.Registry) { - getImportRunner := func(ctx context.Context, gatewayNode int) jobStarter { - startImport := func(c cluster.Cluster) (jobID string, err error) { + getImportRunner := func(ctx context.Context, t test.Test, gatewayNode int) jobStarter { + startImport := func(c cluster.Cluster, t test.Test) (jobID string, err error) { // partsupp is 11.2 GiB. tableName := "partsupp" if c.IsLocal() { @@ -60,7 +60,7 @@ func registerImportNodeShutdown(r registry.Registry) { 'gs://cockroach-fixtures/tpch-csv/sf-100/%[1]s.tbl.8?AUTH=implicit' ) WITH delimiter='|', detached `, tableName) - gatewayDB := c.Conn(ctx, gatewayNode) + gatewayDB := c.Conn(ctx, t.L(), gatewayNode) defer gatewayDB.Close() createStmt, err := readCreateTableFromFixture( @@ -87,10 +87,10 @@ func registerImportNodeShutdown(r registry.Registry) { Cluster: r.MakeClusterSpec(4), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) gatewayNode := 2 nodeToShutdown := 3 - startImport := getImportRunner(ctx, gatewayNode) + startImport := getImportRunner(ctx, t, gatewayNode) jobSurvivesNodeShutdown(ctx, t, c, nodeToShutdown, startImport) }, @@ -101,10 +101,10 @@ func registerImportNodeShutdown(r registry.Registry) { Cluster: r.MakeClusterSpec(4), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) gatewayNode := 2 nodeToShutdown := 2 - startImport := getImportRunner(ctx, gatewayNode) + startImport := getImportRunner(ctx, t, gatewayNode) jobSurvivesNodeShutdown(ctx, t, c, nodeToShutdown, startImport) }, @@ -119,7 +119,7 @@ func registerImportTPCC(r registry.Registry) { c.Put(ctx, t.Cockroach(), "./cockroach") c.Put(ctx, t.DeprecatedWorkload(), "./workload") t.Status("starting csv servers") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) c.Run(ctx, c.All(), `./workload csv-server --port=8081 &> logs/workload-csv-server.log < /dev/null &`) t.Status("running workload") @@ -214,8 +214,8 @@ func registerImportTPCH(r registry.Registry) { // Randomize starting with encryption-at-rest enabled. c.EncryptAtRandom(true) c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) - conn := c.Conn(ctx, 1) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) + conn := c.Conn(ctx, t.L(), 1) if _, err := conn.Exec(`CREATE DATABASE csv;`); err != nil { t.Fatal(err) } @@ -376,7 +376,7 @@ func registerImportDecommissioned(r registry.Registry) { c.Put(ctx, t.Cockroach(), "./cockroach") c.Put(ctx, t.DeprecatedWorkload(), "./workload") t.Status("starting csv servers") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) c.Run(ctx, c.All(), `./workload csv-server --port=8081 &> logs/workload-csv-server.log < /dev/null &`) // Decommission a node. diff --git a/pkg/cmd/roachtest/tests/inconsistency.go b/pkg/cmd/roachtest/tests/inconsistency.go index f0304f47ffdb..4d72c63239fb 100644 --- a/pkg/cmd/roachtest/tests/inconsistency.go +++ b/pkg/cmd/roachtest/tests/inconsistency.go @@ -37,10 +37,10 @@ func runInconsistency(ctx context.Context, t test.Test, c cluster.Cluster) { nodes := c.Range(1, 3) c.Put(ctx, t.Cockroach(), "./cockroach", nodes) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), nodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), nodes) { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) // Disable consistency checks. We're going to be introducing an // inconsistency and wish for it to be detected when we've set up the test // to expect it. @@ -63,9 +63,9 @@ func runInconsistency(ctx context.Context, t test.Test, c cluster.Cluster) { stopOpts := option.DefaultStopOpts() stopOpts.RoachprodOpts.Wait = false stopOpts.RoachprodOpts.Sig = 2 - c.Stop(ctx, stopOpts, nodes) + c.Stop(ctx, t.L(), stopOpts, nodes) stopOpts.RoachprodOpts.Wait = true - c.Stop(ctx, stopOpts, nodes) + c.Stop(ctx, t.L(), stopOpts, nodes) // Write an extraneous transaction record to n1's engine. This means n1 should // ultimately be terminated by the consistency checker (as the other two nodes @@ -98,7 +98,7 @@ func runInconsistency(ctx context.Context, t test.Test, c cluster.Cluster) { // determine why. startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--vmodule=consistency_queue=5,replica_consistency=5,queue=5") - c.Start(ctx, startOpts, install.MakeClusterSettings(), nodes) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), nodes) m.Go(func(ctx context.Context) error { select { case <-time.After(5 * time.Minute): @@ -114,7 +114,7 @@ func runInconsistency(ctx context.Context, t test.Test, c cluster.Cluster) { // check runs against all three nodes. If it targeted only two nodes, a random // one would fatal - not what we want. { - db := c.Conn(ctx, 2) + db := c.Conn(ctx, t.L(), 2) _, err := db.ExecContext(ctx, `SET CLUSTER SETTING server.consistency_check.interval = '10ms'`) if err != nil { t.Fatal(err) @@ -128,7 +128,7 @@ func runInconsistency(ctx context.Context, t test.Test, c cluster.Cluster) { time.Sleep(20 * time.Second) // wait for liveness to time out for dead nodes - db := c.Conn(ctx, 2) + db := c.Conn(ctx, t.L(), 2) rows, err := db.Query(`SELECT node_id FROM crdb_internal.gossip_nodes WHERE is_live = false;`) if err != nil { t.Fatal(err) @@ -151,7 +151,7 @@ func runInconsistency(ctx context.Context, t test.Test, c cluster.Cluster) { c.Run(ctx, c.Node(1), "grep "+ expr+" "+"{log-dir}/cockroach.log") - if err := c.StartE(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(1)); err == nil { + if err := c.StartE(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(1)); err == nil { // NB: we can't easily verify the error because there's a lot of output // which isn't fully included in the error returned from StartE. t.Fatalf("node restart should have failed") diff --git a/pkg/cmd/roachtest/tests/indexes.go b/pkg/cmd/roachtest/tests/indexes.go index 77d4468fa722..3e46b1033aa8 100644 --- a/pkg/cmd/roachtest/tests/indexes.go +++ b/pkg/cmd/roachtest/tests/indexes.go @@ -46,8 +46,8 @@ func registerNIndexes(r registry.Registry, secondaryIndexes int) { c.Put(ctx, t.Cockroach(), "./cockroach", roachNodes) c.Put(ctx, t.DeprecatedWorkload(), "./workload", loadNode) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) - conn := c.Conn(ctx, 1) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) + conn := c.Conn(ctx, t.L(), 1) t.Status("running workload") m := c.NewMonitor(ctx, roachNodes) diff --git a/pkg/cmd/roachtest/tests/inverted_index.go b/pkg/cmd/roachtest/tests/inverted_index.go index 160acef96409..49dfdebdb843 100644 --- a/pkg/cmd/roachtest/tests/inverted_index.go +++ b/pkg/cmd/roachtest/tests/inverted_index.go @@ -42,7 +42,7 @@ func runSchemaChangeInvertedIndex(ctx context.Context, t test.Test, c cluster.Cl c.Put(ctx, t.Cockroach(), "./cockroach", crdbNodes) c.Put(ctx, t.DeprecatedWorkload(), "./workload", workloadNode) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) cmdInit := "./workload init json {pgurl:1}" c.Run(ctx, workloadNode, cmdInit) @@ -66,7 +66,7 @@ func runSchemaChangeInvertedIndex(ctx context.Context, t test.Test, c cluster.Cl m.Go(func(ctx context.Context) error { c.Run(ctx, workloadNode, cmdWrite) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() var count int @@ -93,7 +93,7 @@ func runSchemaChangeInvertedIndex(ctx context.Context, t test.Test, c cluster.Cl }) m.Go(func(ctx context.Context) error { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() t.L().Printf("creating index") diff --git a/pkg/cmd/roachtest/tests/jasyncsql.go b/pkg/cmd/roachtest/tests/jasyncsql.go index 975414c8b77a..f5051cc34325 100644 --- a/pkg/cmd/roachtest/tests/jasyncsql.go +++ b/pkg/cmd/roachtest/tests/jasyncsql.go @@ -31,14 +31,14 @@ func registerJasyncSQL(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/jepsen.go b/pkg/cmd/roachtest/tests/jepsen.go index c0dd3b4ac6b0..9d533256d98a 100644 --- a/pkg/cmd/roachtest/tests/jepsen.go +++ b/pkg/cmd/roachtest/tests/jepsen.go @@ -135,7 +135,7 @@ func initJepsen(ctx context.Context, t test.Test, c cluster.Cluster) { c.Run(ctx, workers, "sh", "-c", `"cat controller_id_rsa.pub >> .ssh/authorized_keys"`) // Prime the known hosts file, and use the unhashed format to // work around JSCH auth error: https://github.com/jepsen-io/jepsen/blob/master/README.md - ips, err := c.InternalIP(ctx, workers) + ips, err := c.InternalIP(ctx, t.L(), workers) if err != nil { t.Fatal(err) } @@ -154,7 +154,7 @@ func runJepsen(ctx context.Context, t test.Test, c cluster.Cluster, testName, ne // Get the IP addresses for all our workers. var nodeFlags []string - ips, err := c.InternalIP(ctx, c.Range(1, c.Spec().NodeCount-1)) + ips, err := c.InternalIP(ctx, t.L(), c.Range(1, c.Spec().NodeCount-1)) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/jobs.go b/pkg/cmd/roachtest/tests/jobs.go index e13acb3753d9..b7989b211428 100644 --- a/pkg/cmd/roachtest/tests/jobs.go +++ b/pkg/cmd/roachtest/tests/jobs.go @@ -24,7 +24,7 @@ import ( "github.com/cockroachdb/errors" ) -type jobStarter func(c cluster.Cluster) (string, error) +type jobStarter func(c cluster.Cluster, t test.Test) (string, error) // jobSurvivesNodeShutdown is a helper that tests that a given job, // running on the specified gatewayNode will still complete successfully @@ -53,7 +53,7 @@ func jobSurvivesNodeShutdown( m.Go(func(ctx context.Context) error { defer close(jobIDCh) - watcherDB := c.Conn(ctx, watcherNode) + watcherDB := c.Conn(ctx, t.L(), watcherNode) defer watcherDB.Close() // Wait for 3x replication to ensure that the cluster @@ -64,7 +64,7 @@ func jobSurvivesNodeShutdown( t.Status("running job") var jobID string - jobID, err := startJob(c) + jobID, err := startJob(c, t) if err != nil { return errors.Wrap(err, "starting the job") } @@ -105,7 +105,7 @@ func jobSurvivesNodeShutdown( } // Check once a second to see if the job has started running. - watcherDB := c.Conn(ctx, watcherNode) + watcherDB := c.Conn(ctx, t.L(), watcherNode) defer watcherDB.Close() timeToWait := time.Second timer := timeutil.Timer{} @@ -140,7 +140,7 @@ func jobSurvivesNodeShutdown( } t.L().Printf(`stopping node %s`, target) - if err := c.StopE(ctx, option.DefaultStopOpts(), target); err != nil { + if err := c.StopE(ctx, t.L(), option.DefaultStopOpts(), target); err != nil { return errors.Wrapf(err, "could not stop node %s", target) } t.L().Printf("stopped node %s", target) @@ -154,7 +154,7 @@ func jobSurvivesNodeShutdown( // NB: the roachtest harness checks that at the end of the test, all nodes // that have data also have a running process. t.Status(fmt.Sprintf("restarting %s (node restart test is done)\n", target)) - if err := c.StartE(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), target); err != nil { + if err := c.StartE(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), target); err != nil { t.Fatal(errors.Wrapf(err, "could not restart node %s", target)) } } diff --git a/pkg/cmd/roachtest/tests/knex.go b/pkg/cmd/roachtest/tests/knex.go index f15876dcf801..8aa346e3bc48 100644 --- a/pkg/cmd/roachtest/tests/knex.go +++ b/pkg/cmd/roachtest/tests/knex.go @@ -41,12 +41,12 @@ func registerKnex(r registry.Registry) { if err := c.PutLibraries(ctx, "./lib"); err != nil { t.Fatal(err) } - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) require.NoError(t, err) - err = alterZoneConfigAndClusterSettings(ctx, version, c, node[0]) + err = alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]) require.NoError(t, err) err = repeatRunE( diff --git a/pkg/cmd/roachtest/tests/kv.go b/pkg/cmd/roachtest/tests/kv.go index 1851ae7aa7dc..adf26cba0382 100644 --- a/pkg/cmd/roachtest/tests/kv.go +++ b/pkg/cmd/roachtest/tests/kv.go @@ -81,9 +81,9 @@ func registerKV(r registry.Registry) { c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(nodes+1)) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.EncryptedStores = opts.encryption - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() if opts.disableLoadSplits { if _, err := db.ExecContext(ctx, "SET CLUSTER SETTING kv.range_split.by_load_enabled = 'false'"); err != nil { @@ -289,9 +289,9 @@ func registerKVContention(r registry.Registry) { // QPS threshold check in the test is guaranteed to fail. settings := install.MakeClusterSettings() settings.Env = append(settings.Env, "COCKROACH_TXN_LIVENESS_HEARTBEAT_MULTIPLIER=600") - c.Start(ctx, option.DefaultStartOpts(), settings, c.Range(1, nodes)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, c.Range(1, nodes)) - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) // Enable request tracing, which is a good tool for understanding // how different transactions are interacting. if _, err := conn.Exec(` @@ -352,7 +352,7 @@ func registerKVQuiescenceDead(r registry.Registry) { nodes := c.Spec().NodeCount - 1 c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, nodes)) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(nodes+1)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) run := func(cmd string, lastDown bool) { n := nodes @@ -368,7 +368,7 @@ func registerKVQuiescenceDead(r registry.Registry) { m.Wait() } - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() WaitFor3XReplication(t, db) @@ -404,14 +404,14 @@ func registerKVQuiescenceDead(r registry.Registry) { }) // Gracefully shut down third node (doesn't matter whether it's graceful or not). c.Run(ctx, c.Node(nodes), "./cockroach quit --insecure --host=:{pgport:3}") - c.Stop(ctx, option.DefaultStopOpts(), c.Node(nodes)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(nodes)) // Measure qps with node down (i.e. without quiescence). qpsOneDown := qps(func() { // Use a different seed to make sure it's not just stepping into the // other earlier kv invocation's footsteps. run(kv+" --seed 2 {pgurl:1}", true) }) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(nodes)) // satisfy dead node detector, even if test fails below + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(nodes)) // satisfy dead node detector, even if test fails below if minFrac, actFrac := 0.8, qpsOneDown/qpsAllUp; actFrac < minFrac { t.Fatalf( @@ -441,9 +441,9 @@ func registerKVGracefulDraining(r registry.Registry) { // failure will likely be thankful for this additional logging. startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--vmodule=store=2,store_rebalancer=2") - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() WaitFor3XReplication(t, db) @@ -489,7 +489,7 @@ func registerKVGracefulDraining(r registry.Registry) { // Before we start shutting down nodes, wait for the performance // of the workload to stabilize at the expected allowed level. - adminURLs, err := c.ExternalAdminUIAddr(ctx, c.Node(1)) + adminURLs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.Node(1)) if err != nil { return err } @@ -575,14 +575,14 @@ func registerKVGracefulDraining(r registry.Registry) { } m.ExpectDeath() c.Run(ctx, c.Node(nodes), "./cockroach quit --insecure --host=:{pgport:3}") - c.Stop(ctx, option.DefaultStopOpts(), c.Node(nodes)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(nodes)) t.Status("letting workload run with one node down") select { case <-ctx.Done(): return nil case <-time.After(1 * time.Minute): } - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(nodes)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(nodes)) m.ResetDeaths() } @@ -657,7 +657,7 @@ func registerKVSplits(r registry.Registry) { settings.Env = append(settings.Env, "COCKROACH_MEMPROF_INTERVAL=1m", "COCKROACH_DISABLE_QUIESCENCE="+strconv.FormatBool(!item.quiesce)) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--cache=256MiB") - c.Start(ctx, startOpts, settings, c.Range(1, nodes)) + c.Start(ctx, t.L(), startOpts, settings, c.Range(1, nodes)) t.Status("running workload") workloadCtx, workloadCancel := context.WithCancel(ctx) @@ -690,7 +690,7 @@ func registerKVScalability(r registry.Registry) { const maxPerNodeConcurrency = 64 for i := nodes; i <= nodes*maxPerNodeConcurrency; i += nodes { c.Wipe(ctx, c.Range(1, nodes)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) t.Status("running workload") m := c.NewMonitor(ctx, c.Range(1, nodes)) @@ -740,13 +740,13 @@ func registerKVRangeLookups(r registry.Registry) { doneWorkload := make(chan struct{}) c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, nodes)) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(nodes+1)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) t.Status("running workload") conns := make([]*gosql.DB, nodes) for i := 0; i < nodes; i++ { - conns[i] = c.Conn(ctx, i+1) + conns[i] = c.Conn(ctx, t.L(), i+1) } defer func() { for i := 0; i < nodes; i++ { @@ -862,9 +862,9 @@ func registerKVMultiStoreWithOverload(r registry.Registry) { c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(nodes+1)) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.StoreCount = 2 - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() // db1 on store1 and db2 on store2. Writes to db2 will overload store2 and // cause admission control to maintain health by queueing these diff --git a/pkg/cmd/roachtest/tests/kvbench.go b/pkg/cmd/roachtest/tests/kvbench.go index a87989e3de0f..971268c6debe 100644 --- a/pkg/cmd/roachtest/tests/kvbench.go +++ b/pkg/cmd/roachtest/tests/kvbench.go @@ -230,13 +230,13 @@ func runKVBench(ctx context.Context, t test.Test, c cluster.Cluster, b kvBenchSp // splitting can significantly change the underlying layout of the table and // affect benchmark results. c.Wipe(ctx, roachNodes) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) time.Sleep(restartWait) // We currently only support one loadGroup. resultChan := make(chan *kvBenchResult, 1) m.Go(func(ctx context.Context) error { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) var initCmd strings.Builder diff --git a/pkg/cmd/roachtest/tests/ledger.go b/pkg/cmd/roachtest/tests/ledger.go index 666f896a2bec..b9a7840dbc9a 100644 --- a/pkg/cmd/roachtest/tests/ledger.go +++ b/pkg/cmd/roachtest/tests/ledger.go @@ -38,7 +38,7 @@ func registerLedger(r registry.Registry) { c.Put(ctx, t.Cockroach(), "./cockroach", roachNodes) c.Put(ctx, t.DeprecatedWorkload(), "./workload", loadNode) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) t.Status("running workload") m := c.NewMonitor(ctx, roachNodes) diff --git a/pkg/cmd/roachtest/tests/libpq.go b/pkg/cmd/roachtest/tests/libpq.go index a843fd7cf179..b591bbf10203 100644 --- a/pkg/cmd/roachtest/tests/libpq.go +++ b/pkg/cmd/roachtest/tests/libpq.go @@ -35,13 +35,13 @@ func registerLibPQ(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/liquibase.go b/pkg/cmd/roachtest/tests/liquibase.go index 8634397dd2bf..a0063c3713bd 100644 --- a/pkg/cmd/roachtest/tests/liquibase.go +++ b/pkg/cmd/roachtest/tests/liquibase.go @@ -36,14 +36,14 @@ func registerLiquibase(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/many_splits.go b/pkg/cmd/roachtest/tests/many_splits.go index 6282ec58b40e..00e29859e9f3 100644 --- a/pkg/cmd/roachtest/tests/many_splits.go +++ b/pkg/cmd/roachtest/tests/many_splits.go @@ -28,9 +28,9 @@ func runManySplits(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") settings := install.MakeClusterSettings() settings.Env = append(settings.Env, "COCKROACH_SCAN_MAX_IDLE_TIME=5ms") - c.Start(ctx, option.DefaultStartOpts(), settings) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() // Wait for upreplication then create many ranges. diff --git a/pkg/cmd/roachtest/tests/mixed_version_decommission.go b/pkg/cmd/roachtest/tests/mixed_version_decommission.go index fdf8aeeffeba..a4cb06f150a2 100644 --- a/pkg/cmd/roachtest/tests/mixed_version_decommission.go +++ b/pkg/cmd/roachtest/tests/mixed_version_decommission.go @@ -266,6 +266,6 @@ func startVersion(nodes option.NodeListOption, version string) versionStep { settings := install.MakeClusterSettings(install.BinaryOption(cockroachBinaryPath(version))) startOpts := option.DefaultStartOpts() startOpts.RoachtestOpts.DontEncrypt = true - u.c.Start(ctx, startOpts, settings, nodes) + u.c.Start(ctx, t.L(), startOpts, settings, nodes) } } diff --git a/pkg/cmd/roachtest/tests/multitenant.go b/pkg/cmd/roachtest/tests/multitenant.go index aa44a22fc6ad..c380e4001b9a 100644 --- a/pkg/cmd/roachtest/tests/multitenant.go +++ b/pkg/cmd/roachtest/tests/multitenant.go @@ -25,15 +25,15 @@ import ( func runAcceptanceMultitenant(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) const tenantID = 123 { - _, err := c.Conn(ctx, 1).Exec(`SELECT crdb_internal.create_tenant($1)`, tenantID) + _, err := c.Conn(ctx, t.L(), 1).Exec(`SELECT crdb_internal.create_tenant($1)`, tenantID) require.NoError(t, err) } - kvAddrs, err := c.ExternalAddr(ctx, c.All()) + kvAddrs, err := c.ExternalAddr(ctx, t.L(), c.All()) require.NoError(t, err) const ( diff --git a/pkg/cmd/roachtest/tests/multitenant_upgrade.go b/pkg/cmd/roachtest/tests/multitenant_upgrade.go index a662b36ac52e..700de33ef9c6 100644 --- a/pkg/cmd/roachtest/tests/multitenant_upgrade.go +++ b/pkg/cmd/roachtest/tests/multitenant_upgrade.go @@ -93,11 +93,11 @@ func (tn *tenantNode) start(ctx context.Context, t test.Test, c cluster.Cluster, tn.httpPort, tn.sqlPort, extraArgs..., ) - externalUrls, err := c.ExternalPGUrl(ctx, c.Node(tn.node)) + externalUrls, err := c.ExternalPGUrl(ctx, t.L(), c.Node(tn.node)) require.NoError(t, err) u, err := url.Parse(externalUrls[0]) require.NoError(t, err) - internalUrls, err := c.ExternalIP(ctx, c.Node(tn.node)) + internalUrls, err := c.ExternalIP(ctx, t.L(), c.Node(tn.node)) require.NoError(t, err) u.Host = internalUrls[0] + ":" + strconv.Itoa(tn.sqlPort) tn.pgURL = u.String() @@ -164,14 +164,14 @@ func runMultiTenantUpgrade(ctx context.Context, t test.Test, c cluster.Cluster, kvNodes := c.Node(1) settings := install.MakeClusterSettings(install.BinaryOption(predecessorBinary)) - c.Start(ctx, option.DefaultStartOpts(), settings, kvNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, kvNodes) - kvAddrs, err := c.ExternalAddr(ctx, kvNodes) + kvAddrs, err := c.ExternalAddr(ctx, t.L(), kvNodes) require.NoError(t, err) const tenant11HTTPPort, tenant11SQLPort = 8011, 20011 const tenant11ID = 11 - runner := sqlutils.MakeSQLRunner(c.Conn(ctx, 1)) + runner := sqlutils.MakeSQLRunner(c.Conn(ctx, t.L(), 1)) // We'll sometimes have to wait out the backoff of the host cluster // auto-update loop (at the time of writing 30s), plus some migrations may be // genuinely long-running. @@ -208,9 +208,9 @@ func runMultiTenantUpgrade(ctx context.Context, t test.Test, c cluster.Cluster, } t.Status("upgrading host server") - c.Stop(ctx, option.DefaultStopOpts(), kvNodes) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), kvNodes) settings.Binary = currentBinary - c.Start(ctx, option.DefaultStartOpts(), settings, kvNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, kvNodes) time.Sleep(time.Second) t.Status("checking the pre-upgrade sql server still works after the KV binary upgrade") diff --git a/pkg/cmd/roachtest/tests/network.go b/pkg/cmd/roachtest/tests/network.go index 4bfc8d90a538..be96cb0037d0 100644 --- a/pkg/cmd/roachtest/tests/network.go +++ b/pkg/cmd/roachtest/tests/network.go @@ -44,9 +44,9 @@ func runNetworkSanity(ctx context.Context, t test.Test, origC cluster.Cluster, n t.Fatal(err) } - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - db := c.Conn(ctx, 1) // unaffected by toxiproxy + db := c.Conn(ctx, t.L(), 1) // unaffected by toxiproxy defer db.Close() WaitFor3XReplication(t, db) @@ -124,8 +124,8 @@ func runNetworkAuthentication(ctx context.Context, t test.Test, c cluster.Cluste // we don't want. Starting all nodes at once ensures // that they use coherent certs. settings := install.MakeClusterSettings(install.SecureOption(true)) - c.Start(ctx, option.DefaultStartOpts(), settings, serverNodes) - require.NoError(t, c.StopE(ctx, option.DefaultStopOpts(), serverNodes)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, serverNodes) + require.NoError(t, c.StopE(ctx, t.L(), option.DefaultStopOpts(), serverNodes)) t.L().Printf("restarting nodes...") // For troubleshooting the test, the engineer can add the following @@ -137,17 +137,17 @@ func runNetworkAuthentication(ctx context.Context, t test.Test, c cluster.Cluste // "--env=COCKROACH_SCAN_MAX_IDLE_TIME=20ms", startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--locality=node=1", "--accept-sql-without-tls") - c.Start(ctx, startOpts, settings, c.Node(1)) + c.Start(ctx, t.L(), startOpts, settings, c.Node(1)) // See comment above about env vars. // "--env=COCKROACH_SCAN_INTERVAL=200ms", // "--env=COCKROACH_SCAN_MAX_IDLE_TIME=20ms", startOpts = option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--locality=node=other", "--accept-sql-without-tls") - c.Start(ctx, startOpts, settings, c.Range(2, n-1)) + c.Start(ctx, t.L(), startOpts, settings, c.Range(2, n-1)) t.L().Printf("retrieving server addresses...") - serverAddrs, err := c.InternalAddr(ctx, serverNodes) + serverAddrs, err := c.InternalAddr(ctx, t.L(), serverNodes) require.NoError(t, err) t.L().Printf("fetching certs...") @@ -168,7 +168,7 @@ func runNetworkAuthentication(ctx context.Context, t test.Test, c cluster.Cluste })) t.L().Printf("connecting to cluster from roachtest...") - db, err := c.ConnE(ctx, 1) + db, err := c.ConnE(ctx, t.L(), 1) require.NoError(t, err) defer db.Close() @@ -377,10 +377,10 @@ func runNetworkTPCC(ctx context.Context, t test.Test, origC cluster.Cluster, nod } const warehouses = 1 - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), serverNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), serverNodes) c.Run(ctx, c.Node(1), tpccImportCmd(warehouses)) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() WaitFor3XReplication(t, db) @@ -414,7 +414,7 @@ func runNetworkTPCC(ctx context.Context, t test.Test, origC cluster.Cluster, nod // us over the threshold. const thresh = 350 - uiAddrs, err := c.ExternalAdminUIAddr(ctx, serverNodes) + uiAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), serverNodes) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/nodejs_postgres.go b/pkg/cmd/roachtest/tests/nodejs_postgres.go index 32bf4173281c..9940fc463a7a 100644 --- a/pkg/cmd/roachtest/tests/nodejs_postgres.go +++ b/pkg/cmd/roachtest/tests/nodejs_postgres.go @@ -44,7 +44,7 @@ func registerNodeJSPostgres(r registry.Registry) { err := c.PutE(ctx, t.L(), t.Cockroach(), "./cockroach", c.All()) require.NoError(t, err) settings := install.MakeClusterSettings(install.SecureOption(true)) - err = c.StartE(ctx, option.DefaultStartOpts(), settings) + err = c.StartE(ctx, t.L(), option.DefaultStartOpts(), settings) require.NoError(t, err) const user = "testuser" // certs auto-generated by roachprod start --secure @@ -60,10 +60,10 @@ func registerNodeJSPostgres(r registry.Registry) { ) require.NoError(t, err) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) require.NoError(t, err) - err = alterZoneConfigAndClusterSettings(ctx, version, c, node[0]) + err = alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]) require.NoError(t, err) err = repeatRunE( diff --git a/pkg/cmd/roachtest/tests/orm_helpers.go b/pkg/cmd/roachtest/tests/orm_helpers.go index bb46c09de832..57cee594be62 100644 --- a/pkg/cmd/roachtest/tests/orm_helpers.go +++ b/pkg/cmd/roachtest/tests/orm_helpers.go @@ -26,9 +26,9 @@ import ( // cause thousands of table descriptors and schema change jobs to accumulate // rapidly, thereby decreasing performance. func alterZoneConfigAndClusterSettings( - ctx context.Context, version string, c cluster.Cluster, nodeIdx int, + ctx context.Context, t test.Test, version string, c cluster.Cluster, nodeIdx int, ) error { - db, err := c.ConnE(ctx, nodeIdx) + db, err := c.ConnE(ctx, t.L(), nodeIdx) if err != nil { return err } diff --git a/pkg/cmd/roachtest/tests/overload_tpcc_olap.go b/pkg/cmd/roachtest/tests/overload_tpcc_olap.go index 94dd33d7d5d7..212cfcf5f567 100644 --- a/pkg/cmd/roachtest/tests/overload_tpcc_olap.go +++ b/pkg/cmd/roachtest/tests/overload_tpcc_olap.go @@ -82,7 +82,7 @@ func verifyNodeLiveness( ctx context.Context, c cluster.Cluster, t test.Test, runDuration time.Duration, ) { const maxFailures = 10 - adminURLs, err := c.ExternalAdminUIAddr(ctx, c.Node(1)) + adminURLs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.Node(1)) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/pgjdbc.go b/pkg/cmd/roachtest/tests/pgjdbc.go index 1773a1507f16..32dfc956898b 100644 --- a/pkg/cmd/roachtest/tests/pgjdbc.go +++ b/pkg/cmd/roachtest/tests/pgjdbc.go @@ -39,14 +39,14 @@ func registerPgjdbc(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/pgx.go b/pkg/cmd/roachtest/tests/pgx.go index 74dcbb4a6c2c..6c5b63266ed3 100644 --- a/pkg/cmd/roachtest/tests/pgx.go +++ b/pkg/cmd/roachtest/tests/pgx.go @@ -40,14 +40,14 @@ func registerPgx(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } @@ -94,7 +94,7 @@ func registerPgx(r registry.Registry) { t.L().Printf("%s", status) t.Status("setting up test db") - db, err := c.ConnE(ctx, node[0]) + db, err := c.ConnE(ctx, t.L(), node[0]) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/pop.go b/pkg/cmd/roachtest/tests/pop.go index e968e8472f29..5c0bd2580aa7 100644 --- a/pkg/cmd/roachtest/tests/pop.go +++ b/pkg/cmd/roachtest/tests/pop.go @@ -34,12 +34,12 @@ func registerPop(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/privilege_version_upgrade.go b/pkg/cmd/roachtest/tests/privilege_version_upgrade.go index 168dfb262f35..65428ef4a6c4 100644 --- a/pkg/cmd/roachtest/tests/privilege_version_upgrade.go +++ b/pkg/cmd/roachtest/tests/privilege_version_upgrade.go @@ -111,7 +111,7 @@ func runPrivilegeVersionUpgrade( func createDatabasePrivilegesStep(privileges privilege.List) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) require.NoError(t, err) _, err = conn.Exec("CREATE DATABASE test") require.NoError(t, err) @@ -126,7 +126,7 @@ func createDatabasePrivilegesStep(privileges privilege.List) versionStep { func checkDatabasePrivilegesStep(privileges privilege.List) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) require.NoError(t, err) r := sqlutils.MakeSQLRunner(conn) @@ -144,7 +144,7 @@ func checkDatabasePrivilegesStep(privileges privilege.List) versionStep { func createSchemaPrivilegesStep(privileges privilege.List) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) require.NoError(t, err) _, err = conn.Exec("CREATE SCHEMA test.test_schema") require.NoError(t, err) @@ -160,7 +160,7 @@ func createSchemaPrivilegesStep(privileges privilege.List) versionStep { func checkSchemaPrivilegesStep(privileges privilege.List) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) require.NoError(t, err) r := sqlutils.MakeSQLRunner(conn) @@ -178,7 +178,7 @@ func checkSchemaPrivilegesStep(privileges privilege.List) versionStep { func createTypePrivilegesStep(privileges privilege.List) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) require.NoError(t, err) _, err = conn.Exec("CREATE TYPE test.test_type AS ENUM()") require.NoError(t, err) @@ -194,7 +194,7 @@ func createTypePrivilegesStep(privileges privilege.List) versionStep { func checkTypePrivilegesStep(privileges privilege.List) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) require.NoError(t, err) r := sqlutils.MakeSQLRunner(conn) @@ -215,7 +215,7 @@ func checkTypePrivilegesStep(privileges privilege.List) versionStep { func createTablePrivilegesStep(privileges privilege.List) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) require.NoError(t, err) _, err = conn.Exec("CREATE TABLE test.test_table()") require.NoError(t, err) @@ -231,7 +231,7 @@ func createTablePrivilegesStep(privileges privilege.List) versionStep { func checkTablePrivilegesStep(privileges privilege.List) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) require.NoError(t, err) r := sqlutils.MakeSQLRunner(conn) @@ -261,7 +261,7 @@ func resetStep() versionStep { func createUserStep() versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) require.NoError(t, err) _, err = conn.Exec("CREATE USER testuser") require.NoError(t, err) diff --git a/pkg/cmd/roachtest/tests/psycopg.go b/pkg/cmd/roachtest/tests/psycopg.go index c1987345e834..4ab710e548d9 100644 --- a/pkg/cmd/roachtest/tests/psycopg.go +++ b/pkg/cmd/roachtest/tests/psycopg.go @@ -38,14 +38,14 @@ func registerPsycopg(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/queue.go b/pkg/cmd/roachtest/tests/queue.go index d16575f49c7b..d7e25f9a4a34 100644 --- a/pkg/cmd/roachtest/tests/queue.go +++ b/pkg/cmd/roachtest/tests/queue.go @@ -45,7 +45,7 @@ func runQueue(ctx context.Context, t test.Test, c cluster.Cluster) { // Distribute programs to the correct nodes and start CockroachDB. c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, dbNodeCount)) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(workloadNode)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, dbNodeCount)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, dbNodeCount)) runQueueWorkload := func(duration time.Duration, initTables bool) { m := c.NewMonitor(ctx, c.Range(1, dbNodeCount)) @@ -75,7 +75,7 @@ func runQueue(ctx context.Context, t test.Test, c cluster.Cluster) { // getQueueScanTime samples the time to run a statement that scans the queue // table. getQueueScanTime := func() time.Duration { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) sampleCount := 5 samples := make([]time.Duration, sampleCount) for i := 0; i < sampleCount; i++ { @@ -102,7 +102,7 @@ func runQueue(ctx context.Context, t test.Test, c cluster.Cluster) { scanTimeBefore := getQueueScanTime() // Set TTL on table queue.queue to 0, so that rows are deleted immediately - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) _, err := db.ExecContext(ctx, `ALTER TABLE queue.queue CONFIGURE ZONE USING gc.ttlseconds = 30`) if err != nil && strings.Contains(err.Error(), "syntax error") { // Pre-2.1 was EXPERIMENTAL. diff --git a/pkg/cmd/roachtest/tests/quit.go b/pkg/cmd/roachtest/tests/quit.go index 7e6eb18f6023..7b50b55940de 100644 --- a/pkg/cmd/roachtest/tests/quit.go +++ b/pkg/cmd/roachtest/tests/quit.go @@ -59,7 +59,7 @@ func (q *quitTest) init(ctx context.Context) { settings := install.MakeClusterSettings(install.EnvOption(q.env)) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, q.args...) - q.c.Start(ctx, startOpts, settings) + q.c.Start(ctx, q.t.L(), startOpts, settings) } func (q *quitTest) Fatal(args ...interface{}) { @@ -103,7 +103,7 @@ func (q *quitTest) restartNode(ctx context.Context, nodeID int) { settings := install.MakeClusterSettings(install.EnvOption(q.env)) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, q.args...) - q.c.Start(ctx, startOpts, settings, q.c.Node(nodeID)) + q.c.Start(ctx, q.t.L(), startOpts, settings, q.c.Node(nodeID)) q.t.L().Printf("waiting for readiness of node %d\n", nodeID) // Now perform a SQL query. This achieves two goals: @@ -111,7 +111,7 @@ func (q *quitTest) restartNode(ctx context.Context, nodeID int) { // - the particular query forces a cluster-wide RPC; which // forces any circuit breaker to trip and re-establish // the RPC connection if needed. - db := q.c.Conn(ctx, nodeID) + db := q.c.Conn(ctx, q.t.L(), nodeID) defer db.Close() if _, err := db.ExecContext(ctx, `TABLE crdb_internal.cluster_sessions`); err != nil { q.Fatal(err) @@ -119,7 +119,7 @@ func (q *quitTest) restartNode(ctx context.Context, nodeID int) { } func (q *quitTest) waitForUpReplication(ctx context.Context) { - db := q.c.Conn(ctx, 1) + db := q.c.Conn(ctx, q.t.L(), 1) defer db.Close() // We'll want rebalancing to be a bit faster than normal, so @@ -160,7 +160,7 @@ func (q *quitTest) runWithTimeout(ctx context.Context, fn func(ctx context.Conte // to transfer all leases. This way, we exercise the iterating code in // quit/node drain. func (q *quitTest) setupIncrementalDrain(ctx context.Context) { - db := q.c.Conn(ctx, 1) + db := q.c.Conn(ctx, q.t.L(), 1) defer db.Close() if _, err := db.ExecContext(ctx, ` SET CLUSTER SETTING server.shutdown.lease_transfer_wait = '10ms'`); err != nil { @@ -176,7 +176,7 @@ SET CLUSTER SETTING server.shutdown.lease_transfer_wait = '10ms'`); err != nil { func (q *quitTest) createRanges(ctx context.Context) { const numRanges = 500 - db := q.c.Conn(ctx, 1) + db := q.c.Conn(ctx, q.t.L(), 1) defer db.Close() if _, err := db.ExecContext(ctx, fmt.Sprintf(` CREATE TABLE t(x, y, PRIMARY KEY(x)) AS SELECT @1, 1 FROM generate_series(1,%[1]d)`, @@ -246,7 +246,7 @@ func (q *quitTest) checkNoLeases(ctx context.Context, nodeID int) { // Get the report via HTTP. // Flag -s is to remove progress on stderr, so that the buffer // contains the JSON of the response and nothing else. - adminAddrs, err := q.c.InternalAdminUIAddr(ctx, q.c.Node(otherNodeID)) + adminAddrs, err := q.c.InternalAdminUIAddr(ctx, q.t.L(), q.c.Node(otherNodeID)) if err != nil { q.Fatal(err) } @@ -334,7 +334,7 @@ func (q *quitTest) checkNoLeases(ctx context.Context, nodeID int) { q.Fatal(err) } - db := q.c.Conn(ctx, otherNodeID) + db := q.c.Conn(ctx, q.t.L(), otherNodeID) defer db.Close() // For good measure, also write to the table. This ensures it // remains available. @@ -361,7 +361,7 @@ func registerQuitTransfersLeases(r registry.Registry) { stopOpts := option.DefaultStopOpts() stopOpts.RoachprodOpts.Sig = 15 stopOpts.RoachprodOpts.Wait = true - c.Stop(ctx, stopOpts, c.Node(nodeID)) // graceful shutdown + c.Stop(ctx, t.L(), stopOpts, c.Node(nodeID)) // graceful shutdown }) @@ -388,7 +388,7 @@ func registerQuitTransfersLeases(r registry.Registry) { // the log. stopOpts := option.DefaultStopOpts() stopOpts.RoachprodOpts.Sig = 1 - c.Stop(ctx, stopOpts, c.Node(nodeID)) + c.Stop(ctx, t.L(), stopOpts, c.Node(nodeID)) // We use SIGKILL to terminate nodes here. Of course, an operator // should not do this and instead terminate with SIGTERM even // after a complete graceful drain. However, what this test is @@ -404,7 +404,7 @@ func registerQuitTransfersLeases(r registry.Registry) { stopOpts = option.DefaultStopOpts() stopOpts.RoachprodOpts.Sig = 9 stopOpts.RoachprodOpts.Wait = true - c.Stop(ctx, stopOpts, c.Node(nodeID)) + c.Stop(ctx, t.L(), stopOpts, c.Node(nodeID)) }) // Same as "drain" above, but issuing the drain command from @@ -432,11 +432,11 @@ func registerQuitTransfersLeases(r registry.Registry) { // to understand what the following signals are for. stopOpts := option.DefaultStopOpts() stopOpts.RoachprodOpts.Sig = 1 - c.Stop(ctx, stopOpts, c.Node(nodeID)) + c.Stop(ctx, t.L(), stopOpts, c.Node(nodeID)) stopOpts.RoachprodOpts.Sig = 9 stopOpts.RoachprodOpts.Wait = true - c.Stop(ctx, stopOpts, c.Node(nodeID)) + c.Stop(ctx, t.L(), stopOpts, c.Node(nodeID)) }) } @@ -456,7 +456,7 @@ func runQuit( stopOpts := option.DefaultStopOpts() stopOpts.RoachprodOpts.Sig = 0 stopOpts.RoachprodOpts.Wait = true - c.Stop(ctx, stopOpts, c.Node(nodeID)) // no shutdown, just wait for exit + c.Stop(ctx, t.L(), stopOpts, c.Node(nodeID)) // no shutdown, just wait for exit return []byte(output) } @@ -506,7 +506,7 @@ func registerQuitAllNodes(r registry.Registry) { settings := install.MakeClusterSettings(install.EnvOption(q.env)) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, q.args...) - q.c.Start(ctx, startOpts, settings) + q.c.Start(ctx, t.L(), startOpts, settings) }, }) } diff --git a/pkg/cmd/roachtest/tests/rapid_restart.go b/pkg/cmd/roachtest/tests/rapid_restart.go index b152855493a2..1043c208f552 100644 --- a/pkg/cmd/roachtest/tests/rapid_restart.go +++ b/pkg/cmd/roachtest/tests/rapid_restart.go @@ -47,7 +47,7 @@ func runRapidRestart(ctx context.Context, t test.Test, c cluster.Cluster) { for i := 0; i < 3; i++ { startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.SkipInit = true - if err := c.StartE(ctx, startOpts, install.MakeClusterSettings(), node); err != nil { + if err := c.StartE(ctx, t.L(), startOpts, install.MakeClusterSettings(), node); err != nil { t.Fatalf("error during start: %v", err) } @@ -61,7 +61,7 @@ func runRapidRestart(ctx context.Context, t test.Test, c cluster.Cluster) { sig := [2]int{2, 9}[rand.Intn(2)] stopOpts := option.DefaultStopOpts() stopOpts.RoachprodOpts.Sig = sig - if err := c.StopE(ctx, stopOpts, node); err != nil { + if err := c.StopE(ctx, t.L(), stopOpts, node); err != nil { t.Fatalf("error during stop: %v", err) } } @@ -73,7 +73,7 @@ func runRapidRestart(ctx context.Context, t test.Test, c cluster.Cluster) { // Verify the cluster is ok by torturing the prometheus endpoint until it // returns success. A side-effect is to prevent regression of #19559. for !done() { - adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, node) + adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), node) if err != nil { t.Fatal(err) } @@ -95,6 +95,6 @@ func runRapidRestart(ctx context.Context, t test.Test, c cluster.Cluster) { // Clean up for the test harness. Usually we want to leave nodes running so // that consistency checks can be run, but in this case there's not much // there in the first place anyway. - c.Stop(ctx, option.DefaultStopOpts(), node) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), node) c.Wipe(ctx, node) } diff --git a/pkg/cmd/roachtest/tests/rebalance_load.go b/pkg/cmd/roachtest/tests/rebalance_load.go index 6a2fd22fc0df..909cd2862bb7 100644 --- a/pkg/cmd/roachtest/tests/rebalance_load.go +++ b/pkg/cmd/roachtest/tests/rebalance_load.go @@ -19,11 +19,11 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/errors" "golang.org/x/sync/errgroup" @@ -60,7 +60,7 @@ func registerRebalanceLoad(r registry.Registry) { c.Put(ctx, t.Cockroach(), "./cockroach", roachNodes) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--vmodule=store_rebalancer=5,allocator=5,allocator_scorer=5,replicate_queue=5") - c.Start(ctx, startOpts, install.MakeClusterSettings(), roachNodes) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), roachNodes) c.Put(ctx, t.DeprecatedWorkload(), "./workload", appNode) c.Run(ctx, appNode, fmt.Sprintf("./workload init kv --drop --splits=%d {pgurl:1}", splits)) @@ -92,7 +92,7 @@ func registerRebalanceLoad(r registry.Registry) { m.Go(func() error { t.Status("checking for lease balance") - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() t.Status("disable load based splitting") diff --git a/pkg/cmd/roachtest/tests/replicagc.go b/pkg/cmd/roachtest/tests/replicagc.go index 9c6125400c0d..9e1ad5dcaaae 100644 --- a/pkg/cmd/roachtest/tests/replicagc.go +++ b/pkg/cmd/roachtest/tests/replicagc.go @@ -63,7 +63,7 @@ func runReplicaGCChangedPeers( c.Put(ctx, t.Cockroach(), "./cockroach") c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(1)) settings := install.MakeClusterSettings(install.EnvOption([]string{"COCKROACH_SCAN_MAX_IDLE_TIME=5ms"})) - c.Start(ctx, option.DefaultStartOpts(), settings, c.Range(1, 3)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, c.Range(1, 3)) h := &replicagcTestHelper{c: c, t: t} @@ -76,10 +76,10 @@ func runReplicaGCChangedPeers( // Kill the third node so it won't know that all of its replicas are moved // elsewhere (we don't use the first because that's what roachprod will // join new nodes to). - c.Stop(ctx, option.DefaultStopOpts(), c.Node(3)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(3)) // Start three new nodes that will take over all data. - c.Start(ctx, option.DefaultStartOpts(), settings, c.Range(4, 6)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, c.Range(4, 6)) // Decommission n1-3, with n3 in absentia, moving the replicas to n4-6. if err := h.decommission(ctx, c.Range(1, 3), 2, "--wait=none"); err != nil { @@ -100,10 +100,10 @@ func runReplicaGCChangedPeers( // // https://github.com/cockroachdb/cockroach/issues/67910#issuecomment-884856356 t.Status("waiting for zero replicas on n3") - waitForZeroReplicasOnN3(ctx, t, c.Conn(ctx, 1)) + waitForZeroReplicasOnN3(ctx, t, c.Conn(ctx, t.L(), 1)) // Stop the remaining two old nodes, no replicas remaining there. - c.Stop(ctx, option.DefaultStopOpts(), c.Range(1, 2)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Range(1, 2)) // Set up zone configs to isolate out nodes with the `deadNodeAttr` // attribute. We'll later start n3 using this attribute to test GC replica @@ -128,28 +128,28 @@ func runReplicaGCChangedPeers( // rebalancing and repair attempts. Lacking this information it does not // do that within the store dead interval (5m, i.e. too long for this // test). - c.Stop(ctx, option.DefaultStopOpts(), c.Range(4, 6)) - c.Start(ctx, option.DefaultStartOpts(), settings, c.Range(4, 6)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Range(4, 6)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, c.Range(4, 6)) } // Restart n3. We have to manually tell it where to find a new node or it // won't be able to connect. Give it the deadNodeAttr attribute that we've // used as a negative constraint for "everything", which should prevent new // replicas from being added to it. - internalAddrs, err := c.InternalAddr(ctx, c.Node(4)) + internalAddrs, err := c.InternalAddr(ctx, t.L(), c.Node(4)) if err != nil { t.Fatal(err) } startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--join="+internalAddrs[0], "--attrs="+deadNodeAttr, "--vmodule=raft=5,replicate_queue=5,allocator=5") - c.Start(ctx, startOpts, settings, c.Node(3)) + c.Start(ctx, t.L(), startOpts, settings, c.Node(3)) // Loop for two metric sample intervals (10s) to make sure n3 doesn't see any // underreplicated ranges. h.waitForZeroReplicas(ctx, 3) // Restart the remaining nodes to satisfy the dead node detector. - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, 2)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, 2)) } type replicagcTestHelper struct { @@ -158,7 +158,7 @@ type replicagcTestHelper struct { } func (h *replicagcTestHelper) waitForFullReplication(ctx context.Context) { - db := h.c.Conn(ctx, 1) + db := h.c.Conn(ctx, h.t.L(), 1) defer func() { _ = db.Close() }() @@ -179,7 +179,7 @@ func (h *replicagcTestHelper) waitForFullReplication(ctx context.Context) { } func (h *replicagcTestHelper) waitForZeroReplicas(ctx context.Context, targetNode int) { - db := h.c.Conn(ctx, targetNode) + db := h.c.Conn(ctx, h.t.L(), targetNode) defer func() { _ = db.Close() }() @@ -243,7 +243,7 @@ func (h *replicagcTestHelper) recommission( // nodes started with deadNodeAttr. This can then be used as a negative // constraint for everything. func (h *replicagcTestHelper) isolateDeadNodes(ctx context.Context, runNode int) { - db := h.c.Conn(ctx, runNode) + db := h.c.Conn(ctx, h.t.L(), runNode) defer func() { _ = db.Close() }() diff --git a/pkg/cmd/roachtest/tests/reset_quorum.go b/pkg/cmd/roachtest/tests/reset_quorum.go index 534130d53e27..5c10360134ae 100644 --- a/pkg/cmd/roachtest/tests/reset_quorum.go +++ b/pkg/cmd/roachtest/tests/reset_quorum.go @@ -34,8 +34,8 @@ func runResetQuorum(ctx context.Context, t test.Test, c cluster.Cluster) { settings := install.MakeClusterSettings(install.EnvOption([]string{"COCKROACH_SCAN_MAX_IDLE_TIME=5ms"})) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--attrs=A") - c.Start(ctx, startOpts, settings, c.Range(1, 5)) - db := c.Conn(ctx, 1) + c.Start(ctx, t.L(), startOpts, settings, c.Range(1, 5)) + db := c.Conn(ctx, t.L(), 1) defer db.Close() rows, err := db.QueryContext(ctx, `SELECT target FROM crdb_internal.zones`) @@ -50,7 +50,7 @@ func runResetQuorum(ctx context.Context, t test.Test, c cluster.Cluster) { startOpts = option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--attrs=B") - c.Start(ctx, startOpts, settings, c.Range(6, 8)) + c.Start(ctx, t.L(), startOpts, settings, c.Range(6, 8)) _, err = db.Exec(`CREATE TABLE lostrange (id INT PRIMARY KEY, v STRING)`) require.NoError(t, err) @@ -104,7 +104,7 @@ OR // Now 'lostrange' is on n6-n8 and nothing else is. The nodes go down // permanently (the wiping prevents the test runner from failing the // test after it has passed - we cannot restart those nodes). - c.Stop(ctx, option.DefaultStopOpts(), c.Range(6, 8)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Range(6, 8)) c.Wipe(ctx, c.Range(6, 8)) // Should not be able to read from it even (generously) after a lease timeout. diff --git a/pkg/cmd/roachtest/tests/restart.go b/pkg/cmd/roachtest/tests/restart.go index e2d995c523f5..cd13394f1852 100644 --- a/pkg/cmd/roachtest/tests/restart.go +++ b/pkg/cmd/roachtest/tests/restart.go @@ -32,7 +32,7 @@ func runRestart(ctx context.Context, t test.Test, c cluster.Cluster, downDuratio c.Put(ctx, t.Cockroach(), "./cockroach", crdbNodes) startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, "--vmodule=raft_log_queue=3") - c.Start(ctx, startOpts, install.MakeClusterSettings(), crdbNodes) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), crdbNodes) // We don't really need tpcc, we just need a good amount of traffic and a good // amount of data. @@ -56,7 +56,7 @@ func runRestart(ctx context.Context, t test.Test, c cluster.Cluster, downDuratio time.Sleep(11 * time.Minute) // Stop a node. - c.Stop(ctx, option.DefaultStopOpts(), c.Node(restartNode)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(restartNode)) // Wait for between 10s and `server.time_until_store_dead` while sending // traffic to one of the nodes that are not down. This used to cause lots of @@ -67,7 +67,7 @@ func runRestart(ctx context.Context, t test.Test, c cluster.Cluster, downDuratio // Bring it back up and make sure it can serve a query within a reasonable // time limit. For now, less time than it was down for. - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(restartNode)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(restartNode)) // Dialing the formerly down node may still be prevented by the circuit breaker // for a short moment (seconds) after n3 restarts. If it happens, the COUNT(*) @@ -78,7 +78,7 @@ func runRestart(ctx context.Context, t test.Test, c cluster.Cluster, downDuratio time.Sleep(15 * time.Second) start := timeutil.Now() - restartNodeDB := c.Conn(ctx, restartNode) + restartNodeDB := c.Conn(ctx, t.L(), restartNode) if _, err := restartNodeDB.Exec(`SELECT count(*) FROM tpcc.order_line`); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/restore.go b/pkg/cmd/roachtest/tests/restore.go index 1e4da36adec5..db7b84e5968b 100644 --- a/pkg/cmd/roachtest/tests/restore.go +++ b/pkg/cmd/roachtest/tests/restore.go @@ -24,12 +24,12 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/ts/tspb" "github.com/cockroachdb/cockroach/pkg/util/httputil" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" @@ -111,7 +111,7 @@ func (hc *HealthChecker) Runner(ctx context.Context) (err error) { tBegin := timeutil.Now() nodeIdx := 1 + rand.Intn(len(hc.nodes)) - db, err := hc.c.ConnE(ctx, nodeIdx) + db, err := hc.c.ConnE(ctx, hc.t.L(), nodeIdx) if err != nil { return err } @@ -228,9 +228,9 @@ func (dul *DiskUsageLogger) Runner(ctx context.Context) error { } func registerRestoreNodeShutdown(r registry.Registry) { makeRestoreStarter := func(ctx context.Context, t test.Test, c cluster.Cluster, gatewayNode int) jobStarter { - return func(c cluster.Cluster) (string, error) { + return func(c cluster.Cluster, t test.Test) (string, error) { t.L().Printf("connecting to gateway") - gatewayDB := c.Conn(ctx, gatewayNode) + gatewayDB := c.Conn(ctx, t.L(), gatewayNode) defer gatewayDB.Close() t.L().Printf("creating bank database") @@ -299,7 +299,7 @@ func registerRestoreNodeShutdown(r registry.Registry) { gatewayNode := 2 nodeToShutdown := 3 c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) jobSurvivesNodeShutdown(ctx, t, c, nodeToShutdown, makeRestoreStarter(ctx, t, c, gatewayNode)) }, @@ -313,7 +313,7 @@ func registerRestoreNodeShutdown(r registry.Registry) { gatewayNode := 2 nodeToShutdown := 2 c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) jobSurvivesNodeShutdown(ctx, t, c, nodeToShutdown, makeRestoreStarter(ctx, t, c, gatewayNode)) }, @@ -396,7 +396,7 @@ func registerRestore(r registry.Registry) { // Randomize starting with encryption-at-rest enabled. c.EncryptAtRandom(true) c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) m := c.NewMonitor(ctx) // Run the disk usage logger in the monitor to guarantee its @@ -462,10 +462,12 @@ func registerRestore(r registry.Registry) { // specified in m. This is particularly useful for verifying that a counter // metric does not exceed some threshold during a test. For example, the // restore and import tests verify that the range merge queue is inactive. -func verifyMetrics(ctx context.Context, c cluster.Cluster, m map[string]float64) error { +func verifyMetrics( + ctx context.Context, t test.Test, c cluster.Cluster, m map[string]float64, +) error { const sample = 10 * time.Second // Query needed information over the timespan of the query. - adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, c.Node(1)) + adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.Node(1)) if err != nil { return err } diff --git a/pkg/cmd/roachtest/tests/roachmart.go b/pkg/cmd/roachtest/tests/roachmart.go index 257337a1c38a..e09685fc5b63 100644 --- a/pkg/cmd/roachtest/tests/roachmart.go +++ b/pkg/cmd/roachtest/tests/roachmart.go @@ -26,7 +26,7 @@ func registerRoachmart(r registry.Registry) { runRoachmart := func(ctx context.Context, t test.Test, c cluster.Cluster, partition bool) { c.Put(ctx, t.Cockroach(), "./cockroach") c.Put(ctx, t.DeprecatedWorkload(), "./workload") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) // TODO(benesch): avoid hardcoding this list. nodes := []struct { diff --git a/pkg/cmd/roachtest/tests/ruby_pg.go b/pkg/cmd/roachtest/tests/ruby_pg.go index 6b4713af18e7..a795d2c3c7f7 100644 --- a/pkg/cmd/roachtest/tests/ruby_pg.go +++ b/pkg/cmd/roachtest/tests/ruby_pg.go @@ -49,14 +49,14 @@ func registerRubyPG(r registry.Registry) { if err := c.PutLibraries(ctx, "./lib"); err != nil { t.Fatal(err) } - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/schema_change_database_version_upgrade.go b/pkg/cmd/roachtest/tests/schema_change_database_version_upgrade.go index 4860058c6e28..1379d77cffe4 100644 --- a/pkg/cmd/roachtest/tests/schema_change_database_version_upgrade.go +++ b/pkg/cmd/roachtest/tests/schema_change_database_version_upgrade.go @@ -55,7 +55,7 @@ func uploadAndStart(nodes option.NodeListOption, v string) versionStep { startOpts.RoachtestOpts.DontEncrypt = true startOpts.RoachprodOpts.Sequential = false settings := install.MakeClusterSettings(install.BinaryOption(binary)) - u.c.Start(ctx, startOpts, settings, nodes) + u.c.Start(ctx, t.L(), startOpts, settings, nodes) } } diff --git a/pkg/cmd/roachtest/tests/schemachange.go b/pkg/cmd/roachtest/tests/schemachange.go index 03aaf957f064..3b2d747e4f92 100644 --- a/pkg/cmd/roachtest/tests/schemachange.go +++ b/pkg/cmd/roachtest/tests/schemachange.go @@ -17,12 +17,12 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/errors" ) @@ -38,8 +38,8 @@ func registerSchemaChangeDuringKV(r registry.Registry) { c.Put(ctx, t.Cockroach(), "./cockroach") c.Put(ctx, t.DeprecatedWorkload(), "./workload") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - db := c.Conn(ctx, 1) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + db := c.Conn(ctx, t.L(), 1) defer db.Close() m := c.NewMonitor(ctx, c.All()) @@ -360,7 +360,7 @@ func makeSchemaChangeBulkIngestTest( c.Put(ctx, t.DeprecatedWorkload(), "./workload", workloadNode) // TODO (lucy): Remove flag once the faster import is enabled by default settings := install.MakeClusterSettings(install.EnvOption([]string{"COCKROACH_IMPORT_WORKLOAD_FASTER=true"})) - c.Start(ctx, option.DefaultStartOpts(), settings, crdbNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings, crdbNodes) // Don't add another index when importing. cmdWrite := fmt.Sprintf( @@ -388,7 +388,7 @@ func makeSchemaChangeBulkIngestTest( }) m.Go(func(ctx context.Context) error { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() if !c.IsLocal() { @@ -480,7 +480,7 @@ func makeSchemaChangeDuringTPCC( func runAndLogStmts( ctx context.Context, t test.Test, c cluster.Cluster, prefix string, stmts []string, ) error { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() t.L().Printf("%s: running %d statements\n", prefix, len(stmts)) start := timeutil.Now() diff --git a/pkg/cmd/roachtest/tests/schemachange_random_load.go b/pkg/cmd/roachtest/tests/schemachange_random_load.go index 58df1bc4939e..1a38cf28b76b 100644 --- a/pkg/cmd/roachtest/tests/schemachange_random_load.go +++ b/pkg/cmd/roachtest/tests/schemachange_random_load.go @@ -137,7 +137,7 @@ func runSchemaChangeRandomLoad( c.Put(ctx, t.DeprecatedWorkload(), "./workload", loadNode) t.Status("starting cockroach nodes") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) c.Run(ctx, loadNode, "./workload init schemachange") result, err := c.RunWithDetailsSingleNode(ctx, t.L(), c.Node(1), "echo", "-n", "{store-dir}") @@ -170,7 +170,7 @@ func runSchemaChangeRandomLoad( // the workload itself (if we even still want it, considering that the // workload itself would be running DROP DATABASE CASCADE). - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() t.Status("performing validation after workload") @@ -186,7 +186,7 @@ func runSchemaChangeRandomLoad( // saveArtifacts saves important test artifacts in the artifacts directory. func saveArtifacts(ctx context.Context, t test.Test, c cluster.Cluster, storeDirectory string) { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() // Save a backup file called schemachange to the store directory. diff --git a/pkg/cmd/roachtest/tests/scrub.go b/pkg/cmd/roachtest/tests/scrub.go index 31d562fa3880..8def4d759283 100644 --- a/pkg/cmd/roachtest/tests/scrub.go +++ b/pkg/cmd/roachtest/tests/scrub.go @@ -72,7 +72,7 @@ func makeScrubTPCCTest( time.Sleep(sleepInterval) } - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) defer conn.Close() t.L().Printf("Starting %d SCRUB checks", numScrubRuns) diff --git a/pkg/cmd/roachtest/tests/sequelize.go b/pkg/cmd/roachtest/tests/sequelize.go index 006e1090d473..5886033e199d 100644 --- a/pkg/cmd/roachtest/tests/sequelize.go +++ b/pkg/cmd/roachtest/tests/sequelize.go @@ -42,19 +42,19 @@ func registerSequelize(r registry.Registry) { if err := c.PutLibraries(ctx, "./lib"); err != nil { t.Fatal(err) } - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } t.Status("create database used by tests") - db, err := c.ConnE(ctx, node[0]) + db, err := c.ConnE(ctx, t.L(), node[0]) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/smoketest_secure.go b/pkg/cmd/roachtest/tests/smoketest_secure.go index 2780a292e9f0..8b283da0b0cf 100644 --- a/pkg/cmd/roachtest/tests/smoketest_secure.go +++ b/pkg/cmd/roachtest/tests/smoketest_secure.go @@ -32,8 +32,8 @@ func registerSecure(r registry.Registry) { Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") settings := install.MakeClusterSettings(install.SecureOption(true)) - c.Start(ctx, option.DefaultStartOpts(), settings) - db := c.Conn(ctx, 1) + c.Start(ctx, t.L(), option.DefaultStartOpts(), settings) + db := c.Conn(ctx, t.L(), 1) defer db.Close() _, err := db.QueryContext(ctx, `SELECT 1`) require.NoError(t, err) diff --git a/pkg/cmd/roachtest/tests/split.go b/pkg/cmd/roachtest/tests/split.go index a265cbab0d44..781f9a918299 100644 --- a/pkg/cmd/roachtest/tests/split.go +++ b/pkg/cmd/roachtest/tests/split.go @@ -135,11 +135,11 @@ func registerLoadSplits(r registry.Registry) { func runLoadSplits(ctx context.Context, t test.Test, c cluster.Cluster, params splitParams) { c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(1)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) m := c.NewMonitor(ctx, c.All()) m.Go(func(ctx context.Context) error { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() t.Status("disable load based splitting") @@ -261,9 +261,9 @@ func runLargeRangeSplits(ctx context.Context, t test.Test, c cluster.Cluster, si c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.All()) numNodes := c.Spec().NodeCount - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(1)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(1)) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() rangeCount := func(t test.Test) (int, string) { @@ -326,7 +326,7 @@ func runLargeRangeSplits(ctx context.Context, t test.Test, c cluster.Cluster, si // Phase 2: add other nodes, wait for full replication of bank table. t.Status("waiting for full replication") { - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(2, numNodes)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(2, numNodes)) m := c.NewMonitor(ctx, c.All()) // NB: we do a round-about thing of making sure that there's at least one // range that has 3 replicas (rather than waiting that there are no ranges diff --git a/pkg/cmd/roachtest/tests/sqlalchemy.go b/pkg/cmd/roachtest/tests/sqlalchemy.go index f30587bf57ea..9e05ff517bd6 100644 --- a/pkg/cmd/roachtest/tests/sqlalchemy.go +++ b/pkg/cmd/roachtest/tests/sqlalchemy.go @@ -138,14 +138,14 @@ func runSQLAlchemy(ctx context.Context, t test.Test, c cluster.Cluster) { t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - version, err := fetchCockroachVersion(ctx, c, node[0]) + version, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, version, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, version, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/sqlsmith.go b/pkg/cmd/roachtest/tests/sqlsmith.go index aa6fc5546b42..d2453f1a7b6e 100644 --- a/pkg/cmd/roachtest/tests/sqlsmith.go +++ b/pkg/cmd/roachtest/tests/sqlsmith.go @@ -93,7 +93,7 @@ func registerSQLSmith(r registry.Registry) { if err := c.PutLibraries(ctx, "./lib"); err != nil { t.Fatalf("could not initialize libraries: %v", err) } - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) setupFunc, ok := setups[setupName] if !ok { @@ -109,7 +109,7 @@ func registerSQLSmith(r registry.Registry) { allConns := make([]*gosql.DB, 0, numNodes) for node := 1; node <= numNodes; node++ { - allConns = append(allConns, c.Conn(ctx, node)) + allConns = append(allConns, c.Conn(ctx, t.L(), node)) } conn := allConns[0] t.Status("executing setup") diff --git a/pkg/cmd/roachtest/tests/sstable_corruption.go b/pkg/cmd/roachtest/tests/sstable_corruption.go index e1b5eb694b27..d90b315d21c1 100644 --- a/pkg/cmd/roachtest/tests/sstable_corruption.go +++ b/pkg/cmd/roachtest/tests/sstable_corruption.go @@ -33,7 +33,7 @@ func runSSTableCorruption(ctx context.Context, t test.Test, c cluster.Cluster) { t.Status("installing cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", crdbNodes) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) { m := c.NewMonitor(ctx, crdbNodes) @@ -49,7 +49,7 @@ func runSSTableCorruption(ctx context.Context, t test.Test, c cluster.Cluster) { m.Wait() } - c.Stop(ctx, option.DefaultStopOpts(), crdbNodes) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), crdbNodes) for _, node := range corruptNodes { result, err := c.RunWithDetailsSingleNode(ctx, t.L(), c.Node(node), @@ -80,7 +80,7 @@ func runSSTableCorruption(ctx context.Context, t test.Test, c cluster.Cluster) { } } - if err := c.StartE(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes); err != nil { + if err := c.StartE(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes); err != nil { // Node detected corruption on start and crashed. This is good. No need // to run workload; the test is complete. _ = c.WipeE(ctx, t.L(), corruptNodes) diff --git a/pkg/cmd/roachtest/tests/status_server.go b/pkg/cmd/roachtest/tests/status_server.go index 51367c5e9b6b..68209e9ac6f6 100644 --- a/pkg/cmd/roachtest/tests/status_server.go +++ b/pkg/cmd/roachtest/tests/status_server.go @@ -29,12 +29,12 @@ import ( func runStatusServer(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) // Get the ids for each node. idMap := make(map[int]roachpb.NodeID) urlMap := make(map[int]string) - adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, c.All()) + adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), c.All()) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/synctest.go b/pkg/cmd/roachtest/tests/synctest.go index f6638e48a94c..fb54754d1c10 100644 --- a/pkg/cmd/roachtest/tests/synctest.go +++ b/pkg/cmd/roachtest/tests/synctest.go @@ -60,7 +60,7 @@ fi c.Run(ctx, n, "mkdir -p {store-dir}/{real,faulty} || true") t.Status("setting up charybdefs") - if err := c.Install(ctx, n, "charybdefs"); err != nil { + if err := c.Install(ctx, t.L(), n, "charybdefs"); err != nil { t.Fatal(err) } c.Run(ctx, n, "sudo charybdefs {store-dir}/faulty -oallow_other,modules=subdir,subdir={store-dir}/real && chmod 777 {store-dir}/{real,faulty}") diff --git a/pkg/cmd/roachtest/tests/sysbench.go b/pkg/cmd/roachtest/tests/sysbench.go index afb152b7d339..25aa37193b1b 100644 --- a/pkg/cmd/roachtest/tests/sysbench.go +++ b/pkg/cmd/roachtest/tests/sysbench.go @@ -97,18 +97,18 @@ func runSysbench(ctx context.Context, t test.Test, c cluster.Cluster, opts sysbe t.Status("installing cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", allNodes) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) - WaitFor3XReplication(t, c.Conn(ctx, allNodes[0])) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) + WaitFor3XReplication(t, c.Conn(ctx, t.L(), allNodes[0])) t.Status("installing haproxy") - if err := c.Install(ctx, loadNode, "haproxy"); err != nil { + if err := c.Install(ctx, t.L(), loadNode, "haproxy"); err != nil { t.Fatal(err) } c.Run(ctx, loadNode, "./cockroach gen haproxy --insecure --url {pgurl:1}") c.Run(ctx, loadNode, "haproxy -f haproxy.cfg -D") t.Status("installing sysbench") - if err := c.Install(ctx, loadNode, "sysbench"); err != nil { + if err := c.Install(ctx, t.L(), loadNode, "sysbench"); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/tlp.go b/pkg/cmd/roachtest/tests/tlp.go index 7b3c43459201..1b033b3229d3 100644 --- a/pkg/cmd/roachtest/tests/tlp.go +++ b/pkg/cmd/roachtest/tests/tlp.go @@ -66,7 +66,7 @@ func runTLP(ctx context.Context, t test.Test, c cluster.Cluster) { fmt.Fprint(tlpLog, "\n\n") } - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) rnd, seed := randutil.NewTestRand() t.L().Printf("seed: %d", seed) @@ -75,7 +75,7 @@ func runTLP(ctx context.Context, t test.Test, c cluster.Cluster) { if err := c.PutLibraries(ctx, "./lib"); err != nil { t.Fatalf("could not initialize libraries: %v", err) } - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) setup := sqlsmith.Setups[sqlsmith.RandTableSetupName](rnd) diff --git a/pkg/cmd/roachtest/tests/toxiproxy.go b/pkg/cmd/roachtest/tests/toxiproxy.go index 3f630bf2ee0b..ba388411c476 100644 --- a/pkg/cmd/roachtest/tests/toxiproxy.go +++ b/pkg/cmd/roachtest/tests/toxiproxy.go @@ -129,7 +129,7 @@ func Toxify( return nil, errors.Wrap(err, "toxify") } - externalAddrs, err := c.ExternalAddr(ctx, n) + externalAddrs, err := c.ExternalAddr(ctx, t.L(), n) if err != nil { return nil, err } @@ -180,7 +180,7 @@ func (tc *ToxiCluster) Proxy(i int) *toxiproxy.Proxy { func (tc *ToxiCluster) ExternalAddr( ctx context.Context, node option.NodeListOption, ) ([]string, error) { - return tc.Cluster.ExternalAddr(ctx, node) + return tc.Cluster.ExternalAddr(ctx, tc.t.L(), node) } // PoisonedExternalAddr gives the external host:port of the toxiproxy process @@ -210,7 +210,7 @@ func (tc *ToxiCluster) PoisonedPGAddr( ) ([]string, error) { var out []string - urls, err := tc.ExternalPGUrl(ctx, node) + urls, err := tc.ExternalPGUrl(ctx, tc.t.L(), node) if err != nil { return nil, err } diff --git a/pkg/cmd/roachtest/tests/tpc_utils.go b/pkg/cmd/roachtest/tests/tpc_utils.go index 6716b18f25c2..c92f607ad19d 100644 --- a/pkg/cmd/roachtest/tests/tpc_utils.go +++ b/pkg/cmd/roachtest/tests/tpc_utils.go @@ -37,7 +37,7 @@ func loadTPCHDataset( m cluster.Monitor, roachNodes option.NodeListOption, ) error { - db := c.Conn(ctx, roachNodes[0]) + db := c.Conn(ctx, t.L(), roachNodes[0]) defer db.Close() if _, err := db.ExecContext(ctx, `USE tpch`); err == nil { @@ -68,7 +68,7 @@ func loadTPCHDataset( // cluster and restore. m.ExpectDeaths(int32(c.Spec().NodeCount)) c.Wipe(ctx, roachNodes) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) m.ResetDeaths() } else if pqErr := (*pq.Error)(nil); !(errors.As(err, &pqErr) && pgcode.MakeCode(string(pqErr.Code)) == pgcode.InvalidCatalogName) { diff --git a/pkg/cmd/roachtest/tests/tpcc.go b/pkg/cmd/roachtest/tests/tpcc.go index 002bdaad8e53..d05c75c1d232 100644 --- a/pkg/cmd/roachtest/tests/tpcc.go +++ b/pkg/cmd/roachtest/tests/tpcc.go @@ -133,15 +133,15 @@ func setupTPCC( // We still use bare workload, though we could likely replace // those with ./cockroach workload as well. c.Put(ctx, t.DeprecatedWorkload(), "./workload", workloadNode) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), crdbNodes) } } func() { opts.Start(ctx, t, c) - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() - WaitFor3XReplication(t, c.Conn(ctx, crdbNodes[0])) + WaitFor3XReplication(t, c.Conn(ctx, t.L(), crdbNodes[0])) switch opts.SetupType { case usingExistingData: // Do nothing. @@ -588,7 +588,7 @@ func registerTPCC(r registry.Registry) { prometheusNode option.NodeListOption, workloadInstances []workloadInstance, ) (tpccChaosEventProcessor, error) { - prometheusNodeIP, err := c.ExternalIP(ctx, prometheusNode) + prometheusNodeIP, err := c.ExternalIP(ctx, t.L(), prometheusNode) if err != nil { return tpccChaosEventProcessor{}, err } @@ -920,7 +920,7 @@ func loadTPCCBench( b tpccBenchSpec, roachNodes, loadNode option.NodeListOption, ) error { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() // Check if the dataset already exists and is already large enough to @@ -943,7 +943,7 @@ func loadTPCCBench( // before restoring. c.Wipe(ctx, roachNodes) startOpts, settings := b.startOpts() - c.Start(ctx, startOpts, settings, roachNodes) + c.Start(ctx, t.L(), startOpts, settings, roachNodes) } else if pqErr := (*pq.Error)(nil); !(errors.As(err, &pqErr) && pgcode.MakeCode(string(pqErr.Code)) == pgcode.InvalidCatalogName) { return err @@ -1038,7 +1038,7 @@ func runTPCCBench(ctx context.Context, t test.Test, c cluster.Cluster, b tpccBen c.EncryptDefault(false) c.EncryptAtRandom(false) startOpts, settings := b.startOpts() - c.Start(ctx, startOpts, settings, roachNodes) + c.Start(ctx, t.L(), startOpts, settings, roachNodes) SetAdmissionControl(ctx, t, c, !b.AdmissionControlDisabled) useHAProxy := b.Chaos const restartWait = 15 * time.Second @@ -1051,7 +1051,7 @@ func runTPCCBench(ctx context.Context, t test.Test, c cluster.Cluster, b tpccBen t.Fatal("distributed chaos benchmarking not supported") } t.Status("installing haproxy") - if err := c.Install(ctx, loadNodes, "haproxy"); err != nil { + if err := c.Install(ctx, t.L(), loadNodes, "haproxy"); err != nil { t.Fatal(err) } c.Run(ctx, loadNodes, "./cockroach gen haproxy --insecure --url {pgurl:1}") @@ -1094,7 +1094,7 @@ func runTPCCBench(ctx context.Context, t test.Test, c cluster.Cluster, b tpccBen // passing warehouse count, making the line search sensitive to the choice // of starting warehouses. Do a best-effort at waiting for the cloud VM(s) // to recover without failing the line search. - if err := c.Reset(ctx); err != nil { + if err := c.Reset(ctx, t.L()); err != nil { // Reset() can flake sometimes, see for example: // https://github.com/cockroachdb/cockroach/issues/61981#issuecomment-826838740 t.L().Printf("failed to reset VMs, proceeding anyway: %s", err) @@ -1106,7 +1106,7 @@ func runTPCCBench(ctx context.Context, t test.Test, c cluster.Cluster, b tpccBen t.Fatal(err) } shortCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) - if err := c.StopE(shortCtx, option.DefaultStopOpts(), roachNodes); err != nil { + if err := c.StopE(shortCtx, t.L(), option.DefaultStopOpts(), roachNodes); err != nil { cancel() t.L().Printf("unable to stop cluster; retrying to allow vm to recover: %s", err) // We usually spend a long time blocking in StopE anyway, but just in case @@ -1128,7 +1128,7 @@ func runTPCCBench(ctx context.Context, t test.Test, c cluster.Cluster, b tpccBen } startOpts, settings := b.startOpts() - c.Start(ctx, startOpts, settings, roachNodes) + c.Start(ctx, t.L(), startOpts, settings, roachNodes) SetAdmissionControl(ctx, t, c, !b.AdmissionControlDisabled) } @@ -1467,6 +1467,7 @@ func setupPrometheus( ctx, *cfg, c, + t.L(), func(ctx context.Context, nodes option.NodeListOption, operation string, args ...string) error { return repeatRunE( ctx, diff --git a/pkg/cmd/roachtest/tests/tpcdsvec.go b/pkg/cmd/roachtest/tests/tpcdsvec.go index 28626f446ba4..8b99f580fa20 100644 --- a/pkg/cmd/roachtest/tests/tpcdsvec.go +++ b/pkg/cmd/roachtest/tests/tpcdsvec.go @@ -58,9 +58,9 @@ func registerTPCDSVec(r registry.Registry) { runTPCDSVec := func(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) - clusterConn := c.Conn(ctx, 1) + clusterConn := c.Conn(ctx, t.L(), 1) disableAutoStats(t, clusterConn) t.Status("restoring TPCDS dataset for Scale Factor 1") if _, err := clusterConn.Exec( @@ -84,7 +84,7 @@ func registerTPCDSVec(r registry.Registry) { // We additionally open fresh connections for each query. setStmtTimeout := fmt.Sprintf("SET statement_timeout='%s';", timeout) firstNode := c.Node(1) - urls, err := c.ExternalPGUrl(ctx, firstNode) + urls, err := c.ExternalPGUrl(ctx, t.L(), firstNode) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/tpce.go b/pkg/cmd/roachtest/tests/tpce.go index 5bb76ca7d6e9..d953040b7e59 100644 --- a/pkg/cmd/roachtest/tests/tpce.go +++ b/pkg/cmd/roachtest/tests/tpce.go @@ -47,16 +47,16 @@ func registerTPCE(r registry.Registry) { startOpts := option.DefaultStartOpts() startOpts.RoachprodOpts.StoreCount = opts.ssds settings := install.MakeClusterSettings(install.NumRacksOption(racks)) - c.Start(ctx, startOpts, settings, roachNodes) + c.Start(ctx, t.L(), startOpts, settings, roachNodes) t.Status("installing docker") - if err := c.Install(ctx, loadNode, "docker"); err != nil { + if err := c.Install(ctx, t.L(), loadNode, "docker"); err != nil { t.Fatal(err) } // Configure to increase the speed of the import. func() { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() if _, err := db.ExecContext( ctx, "SET CLUSTER SETTING kv.bulk_io_write.concurrent_addsstable_requests = $1", 4*opts.ssds, @@ -74,7 +74,7 @@ func registerTPCE(r registry.Registry) { m.Go(func(ctx context.Context) error { const dockerRun = `sudo docker run cockroachdb/tpc-e:latest` - roachNodeIPs, err := c.InternalIP(ctx, roachNodes) + roachNodeIPs, err := c.InternalIP(ctx, t.L(), roachNodes) if err != nil { return err } diff --git a/pkg/cmd/roachtest/tests/tpch_concurrency.go b/pkg/cmd/roachtest/tests/tpch_concurrency.go index 71921261146d..73006c73bee0 100644 --- a/pkg/cmd/roachtest/tests/tpch_concurrency.go +++ b/pkg/cmd/roachtest/tests/tpch_concurrency.go @@ -29,11 +29,11 @@ func registerTPCHConcurrency(r registry.Registry) { setupCluster := func(ctx context.Context, t test.Test, c cluster.Cluster, disableTxnStatsSampling bool) { c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, numNodes-1)) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(numNodes)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, numNodes-1)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, numNodes-1)) // In order to keep more things constant throughout the different test // runs, we disable range merges and range movements. - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) if _, err := conn.Exec("SET CLUSTER SETTING kv.allocator.min_lease_transfer_interval = '24h';"); err != nil { t.Fatal(err) } @@ -51,9 +51,9 @@ func registerTPCHConcurrency(r registry.Registry) { } } - restartCluster := func(ctx context.Context, c cluster.Cluster) { - c.Stop(ctx, option.DefaultStopOpts(), c.Range(1, numNodes-1)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, numNodes-1)) + restartCluster := func(ctx context.Context, c cluster.Cluster, t test.Test) { + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Range(1, numNodes-1)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, numNodes-1)) } // checkConcurrency returns an error if at least one node of the cluster @@ -64,11 +64,11 @@ func registerTPCHConcurrency(r registry.Registry) { // iteration. _ = c.RunE(ctx, c.Node(numNodes), "killall workload") - restartCluster(ctx, c) + restartCluster(ctx, c, t) // Scatter the ranges so that a poor initial placement (after loading // the data set) doesn't impact the results much. - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) if _, err := conn.Exec("USE tpch;"); err != nil { t.Fatal(err) } @@ -77,7 +77,7 @@ func registerTPCHConcurrency(r registry.Registry) { // Populate the range cache on each node. for node := 1; node < numNodes; node++ { - conn = c.Conn(ctx, node) + conn = c.Conn(ctx, t.L(), node) if _, err := conn.Exec("USE tpch;"); err != nil { t.Fatal(err) } @@ -166,7 +166,7 @@ func registerTPCHConcurrency(r registry.Registry) { } // Restart the cluster so that if any nodes crashed in the last // iteration, it doesn't fail the test. - restartCluster(ctx, c) + restartCluster(ctx, c, t) t.Status(fmt.Sprintf("max supported concurrency is %d", minConcurrency)) // Write the concurrency number into the stats.json file to be used by // the roachperf. diff --git a/pkg/cmd/roachtest/tests/tpchbench.go b/pkg/cmd/roachtest/tests/tpchbench.go index 56e38a7c76e1..fb2bf263b78b 100644 --- a/pkg/cmd/roachtest/tests/tpchbench.go +++ b/pkg/cmd/roachtest/tests/tpchbench.go @@ -68,7 +68,7 @@ func runTPCHBench(ctx context.Context, t test.Test, c cluster.Cluster, b tpchBen } t.Status("starting nodes") - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), roachNodes) m := c.NewMonitor(ctx, roachNodes) m.Go(func(ctx context.Context) error { diff --git a/pkg/cmd/roachtest/tests/tpchvec.go b/pkg/cmd/roachtest/tests/tpchvec.go index 8f1d7293d685..b2546c2ad796 100644 --- a/pkg/cmd/roachtest/tests/tpchvec.go +++ b/pkg/cmd/roachtest/tests/tpchvec.go @@ -258,7 +258,7 @@ func (p *tpchVecPerfTest) postTestRunHook( // however, the session variables might contain the old values, // so we will open up new connections for each of the setups in // order to get the correct cluster setup on each. - tempConn := c.Conn(ctx, 1) + tempConn := c.Conn(ctx, t.L(), 1) defer tempConn.Close() if _, err := tempConn.Exec("USE tpch;"); err != nil { t.Fatal(err) @@ -545,9 +545,9 @@ func runTPCHVec( firstNode := c.Node(1) c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) c.Put(ctx, t.DeprecatedWorkload(), "./workload", firstNode) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings()) - conn := c.Conn(ctx, 1) + conn := c.Conn(ctx, t.L(), 1) disableAutoStats(t, conn) t.Status("restoring TPCH dataset for Scale Factor 1") if err := loadTPCHDataset(ctx, t, c, 1 /* sf */, c.NewMonitor(ctx), c.All()); err != nil { diff --git a/pkg/cmd/roachtest/tests/ts_util.go b/pkg/cmd/roachtest/tests/ts_util.go index 79cb36c17d36..db3c5957cfc0 100644 --- a/pkg/cmd/roachtest/tests/ts_util.go +++ b/pkg/cmd/roachtest/tests/ts_util.go @@ -101,7 +101,7 @@ func verifyTxnPerSecond( txnTarget, maxPercentTimeUnderTarget float64, ) { // Query needed information over the timespan of the query. - adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, adminNode) + adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), adminNode) if err != nil { t.Fatal(err) } @@ -152,7 +152,7 @@ func verifyLookupsPerSec( rangeLookupsTarget float64, ) { // Query needed information over the timespan of the query. - adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, adminNode) + adminUIAddrs, err := c.ExternalAdminUIAddr(ctx, t.L(), adminNode) if err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/typeorm.go b/pkg/cmd/roachtest/tests/typeorm.go index 6bd0d0134e9d..58a4446a4634 100644 --- a/pkg/cmd/roachtest/tests/typeorm.go +++ b/pkg/cmd/roachtest/tests/typeorm.go @@ -39,14 +39,14 @@ func registerTypeORM(r registry.Registry) { node := c.Node(1) t.Status("setting up cockroach") c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.All()) - cockroachVersion, err := fetchCockroachVersion(ctx, c, node[0]) + cockroachVersion, err := fetchCockroachVersion(ctx, t.L(), c, node[0]) if err != nil { t.Fatal(err) } - if err := alterZoneConfigAndClusterSettings(ctx, cockroachVersion, c, node[0]); err != nil { + if err := alterZoneConfigAndClusterSettings(ctx, t, cockroachVersion, c, node[0]); err != nil { t.Fatal(err) } diff --git a/pkg/cmd/roachtest/tests/util.go b/pkg/cmd/roachtest/tests/util.go index 5e177b217a37..92dfcb4af6d9 100644 --- a/pkg/cmd/roachtest/tests/util.go +++ b/pkg/cmd/roachtest/tests/util.go @@ -97,7 +97,7 @@ func WaitForUpdatedReplicationReport(ctx context.Context, t test.Test, db *gosql // SetAdmissionControl sets the admission control cluster settings on the // given cluster. func SetAdmissionControl(ctx context.Context, t test.Test, c cluster.Cluster, enabled bool) { - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() val := "true" if !enabled { diff --git a/pkg/cmd/roachtest/tests/util_disk_usage.go b/pkg/cmd/roachtest/tests/util_disk_usage.go index a6d40b9970fa..921857071bce 100644 --- a/pkg/cmd/roachtest/tests/util_disk_usage.go +++ b/pkg/cmd/roachtest/tests/util_disk_usage.go @@ -16,8 +16,8 @@ import ( "strings" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" ) // getDiskUsageInBytes does what's on the tin. nodeIdx starts at one. diff --git a/pkg/cmd/roachtest/tests/version.go b/pkg/cmd/roachtest/tests/version.go index 9a99b236e0e8..d7dc7f7e8a77 100644 --- a/pkg/cmd/roachtest/tests/version.go +++ b/pkg/cmd/roachtest/tests/version.go @@ -41,7 +41,7 @@ func registerVersion(r registry.Registry) { // TODO(mberhault): allow it once version >= 2.1. startOpts := option.DefaultStartOpts() startOpts.RoachtestOpts.DontEncrypt = true - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(1, nodes)) stageDuration := 10 * time.Minute buffer := 10 * time.Minute @@ -88,7 +88,7 @@ func registerVersion(r registry.Registry) { // Make sure everyone is still running. for i := 1; i <= nodes; i++ { t.WorkerStatus("checking ", i) - db := c.Conn(ctx, i) + db := c.Conn(ctx, t.L(), i) defer db.Close() rows, err := db.Query(`SHOW DATABASES`) if err != nil { @@ -112,7 +112,7 @@ func registerVersion(r registry.Registry) { return nil } - db := c.Conn(ctx, 1) + db := c.Conn(ctx, t.L(), 1) defer db.Close() // See analogous comment in the upgrade/mixedWith roachtest. db.SetMaxIdleConns(0) @@ -125,7 +125,7 @@ func registerVersion(r registry.Registry) { stop := func(node int) error { m.ExpectDeath() l.Printf("stopping node %d\n", node) - return c.StopCockroachGracefullyOnNode(ctx, node) + return c.StopCockroachGracefullyOnNode(ctx, t.L(), node) } var oldVersion string @@ -143,7 +143,7 @@ func registerVersion(r registry.Registry) { } c.Put(ctx, t.Cockroach(), "./cockroach", c.Node(i)) - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Node(i)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(i)) if err := sleepAndCheck(); err != nil { return err } @@ -167,7 +167,7 @@ func registerVersion(r registry.Registry) { // Do upgrade for the last node. l.Printf("upgrading last node\n") c.Put(ctx, t.Cockroach(), "./cockroach", c.Node(nodes)) - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Node(nodes)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(nodes)) if err := sleepAndCheck(); err != nil { return err } @@ -182,7 +182,7 @@ func registerVersion(r registry.Registry) { if err := c.Stage(ctx, t.L(), "release", "v"+binaryVersion, "", c.Node(i)); err != nil { t.Fatal(err) } - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Node(i)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(i)) if err := sleepAndCheck(); err != nil { return err } @@ -196,7 +196,7 @@ func registerVersion(r registry.Registry) { return err } c.Put(ctx, t.Cockroach(), "./cockroach", c.Node(i)) - c.Start(ctx, startOpts, install.MakeClusterSettings(), c.Node(i)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(i)) if err := sleepAndCheck(); err != nil { return err } diff --git a/pkg/cmd/roachtest/tests/version_upgrade_public_schema.go b/pkg/cmd/roachtest/tests/version_upgrade_public_schema.go index 3ce3a968bfc9..ed54c392d137 100644 --- a/pkg/cmd/roachtest/tests/version_upgrade_public_schema.go +++ b/pkg/cmd/roachtest/tests/version_upgrade_public_schema.go @@ -100,7 +100,7 @@ func runVersionUpgradePublicSchema( func createDatabaseStep(dbName string) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) defer func() { _ = conn.Close() }() @@ -112,7 +112,7 @@ func createDatabaseStep(dbName string) versionStep { func createTableInDatabasePublicSchema(dbName string) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) defer func() { _ = conn.Close() }() @@ -124,7 +124,7 @@ func createTableInDatabasePublicSchema(dbName string) versionStep { func dropTableInDatabase(dbName string) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) defer func() { _ = conn.Close() }() @@ -136,7 +136,7 @@ func dropTableInDatabase(dbName string) versionStep { func insertIntoTable(dbName string) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) defer func() { _ = conn.Close() }() @@ -148,7 +148,7 @@ func insertIntoTable(dbName string) versionStep { func selectFromTable(dbName string) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) defer func() { _ = conn.Close() }() @@ -171,7 +171,7 @@ func selectFromTable(dbName string) versionStep { func tryReparentingDatabase(shouldError bool, errRe string) versionStep { return func(ctx context.Context, t test.Test, u *versionUpgradeTest) { - conn, err := u.c.ConnE(ctx, loadNode) + conn, err := u.c.ConnE(ctx, t.L(), loadNode) defer func() { _ = conn.Close() }() diff --git a/pkg/cmd/roachtest/tests/versionupgrade.go b/pkg/cmd/roachtest/tests/versionupgrade.go index 2eabe52e6797..d908a0f2e8d1 100644 --- a/pkg/cmd/roachtest/tests/versionupgrade.go +++ b/pkg/cmd/roachtest/tests/versionupgrade.go @@ -235,7 +235,7 @@ func checkpointName(binaryVersion string) string { return "checkpoint-v" + binar func (u *versionUpgradeTest) conn(ctx context.Context, t test.Test, i int) *gosql.DB { if u.conns == nil { for _, i := range u.c.All() { - u.conns = append(u.conns, u.c.Conn(ctx, i)) + u.conns = append(u.conns, u.c.Conn(ctx, t.L(), i)) } } db := u.conns[i-1] @@ -373,7 +373,7 @@ func uploadAndStartFromCheckpointFixture(nodes option.NodeListOption, v string) // NB: can't start sequentially since cluster already bootstrapped. startOpts.RoachprodOpts.Sequential = false startOpts.RoachtestOpts.DontEncrypt = true - u.c.Start(ctx, startOpts, settings, nodes) + u.c.Start(ctx, t.L(), startOpts, settings, nodes) } } @@ -416,13 +416,13 @@ func upgradeNodes( newVersionMsg = "" } t.L().Printf("restarting node %d into version %s", node, newVersionMsg) - c.Stop(ctx, option.DefaultStopOpts(), c.Node(node)) + c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(node)) binary := uploadVersion(ctx, t, c, c.Node(node), newVersion) settings := install.MakeClusterSettings(install.BinaryOption(binary)) startOpts := option.DefaultStartOpts() startOpts.RoachtestOpts.DontEncrypt = true - c.Start(ctx, startOpts, settings, c.Node(node)) + c.Start(ctx, t.L(), startOpts, settings, c.Node(node)) } } @@ -562,8 +562,8 @@ func makeVersionFixtureAndFatal( ) { var useLocalBinary bool if makeFixtureVersion == "" { - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(1)) - require.NoError(t, c.Conn(ctx, 1).QueryRowContext( + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Node(1)) + require.NoError(t, c.Conn(ctx, t.L(), 1).QueryRowContext( ctx, `select regexp_extract(value, '^v([0-9]+\.[0-9]+\.[0-9]+)') from crdb_internal.node_build_info where field = 'Version';`, ).Scan(&makeFixtureVersion)) @@ -621,7 +621,7 @@ func makeVersionFixtureAndFatal( // 2.1 binary, but not the 19.1 binary (as 19.1 and 2.0 are not // compatible). name := checkpointName(u.binaryVersion(ctx, t, 1).String()) - u.c.Stop(ctx, option.DefaultStopOpts(), c.All()) + u.c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.All()) c.Run(ctx, c.All(), binaryPathFromVersion(makeFixtureVersion), "debug", "pebble", "db", "checkpoint", "{store-dir}", "{store-dir}/"+name) diff --git a/pkg/cmd/roachtest/tests/ycsb.go b/pkg/cmd/roachtest/tests/ycsb.go index 802b09d17250..d366af4509eb 100644 --- a/pkg/cmd/roachtest/tests/ycsb.go +++ b/pkg/cmd/roachtest/tests/ycsb.go @@ -58,8 +58,8 @@ func registerYCSB(r registry.Registry) { c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, nodes)) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(nodes+1)) - c.Start(ctx, option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) - WaitFor3XReplication(t, c.Conn(ctx, 1)) + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, nodes)) + WaitFor3XReplication(t, c.Conn(ctx, t.L(), 1)) t.Status("running workload") m := c.NewMonitor(ctx, c.Range(1, nodes)) diff --git a/pkg/cmd/roachtest/work_pool.go b/pkg/cmd/roachtest/work_pool.go index 0e3516d73e29..2475ff9341dc 100644 --- a/pkg/cmd/roachtest/work_pool.go +++ b/pkg/cmd/roachtest/work_pool.go @@ -15,9 +15,9 @@ import ( "fmt" "math" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/quotapool" "github.com/cockroachdb/cockroach/pkg/util/syncutil" diff --git a/pkg/roachprod/BUILD.bazel b/pkg/roachprod/BUILD.bazel index d6cb48d9f2ae..853ac9f5377e 100644 --- a/pkg/roachprod/BUILD.bazel +++ b/pkg/roachprod/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//pkg/roachprod/cloud", "//pkg/roachprod/config", "//pkg/roachprod/install", + "//pkg/roachprod/logger", "//pkg/roachprod/vm", "//pkg/roachprod/vm/aws", "//pkg/roachprod/vm/azure", diff --git a/pkg/roachprod/cloud/BUILD.bazel b/pkg/roachprod/cloud/BUILD.bazel index bb19921c42bd..64353ead7694 100644 --- a/pkg/roachprod/cloud/BUILD.bazel +++ b/pkg/roachprod/cloud/BUILD.bazel @@ -11,6 +11,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/roachprod/config", + "//pkg/roachprod/logger", "//pkg/roachprod/vm", "//pkg/util/log", "//pkg/util/timeutil", diff --git a/pkg/roachprod/cloud/cluster_cloud.go b/pkg/roachprod/cloud/cluster_cloud.go index 18ed5b0eb558..690ecbf96c3e 100644 --- a/pkg/roachprod/cloud/cluster_cloud.go +++ b/pkg/roachprod/cloud/cluster_cloud.go @@ -19,6 +19,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/errors" "golang.org/x/sync/errgroup" @@ -138,20 +139,20 @@ func (c *Cluster) String() string { } // PrintDetails TODO(peter): document -func (c *Cluster) PrintDetails() { - fmt.Printf("%s: %s ", c.Name, c.Clouds()) +func (c *Cluster) PrintDetails(logger *logger.Logger) { + logger.Printf("%s: %s ", c.Name, c.Clouds()) if !c.IsLocal() { l := c.LifetimeRemaining().Round(time.Second) if l <= 0 { - fmt.Printf("expired %s ago\n", -l) + logger.Printf("expired %s ago", -l) } else { - fmt.Printf("%s remaining\n", l) + logger.Printf("%s remaining", l) } } else { - fmt.Printf("(no expiration)\n") + logger.Printf("(no expiration)") } for _, vm := range c.VMs { - fmt.Printf(" %s\t%s\t%s\t%s\n", vm.Name, vm.DNS, vm.PrivateIP, vm.PublicIP) + logger.Printf(" %s\t%s\t%s\t%s", vm.Name, vm.DNS, vm.PrivateIP, vm.PublicIP) } } @@ -177,7 +178,7 @@ func namesFromVM(v vm.VM) (userName string, clusterName string, _ error) { // ListCloud returns information about all instances (across all available // providers). -func ListCloud() (*Cloud, error) { +func ListCloud(l *logger.Logger) (*Cloud, error) { cloud := &Cloud{ Clusters: make(Clusters), } @@ -191,7 +192,7 @@ func ListCloud() (*Cloud, error) { provider := vm.Providers[providerName] g.Go(func() error { var err error - providerVMs[index], err = provider.List() + providerVMs[index], err = provider.List(l) return err }) } @@ -247,7 +248,10 @@ func ListCloud() (*Cloud, error) { // CreateCluster TODO(peter): document func CreateCluster( - nodes int, opts vm.CreateOpts, providerOptsContainer vm.ProviderOptionsContainer, + l *logger.Logger, + nodes int, + opts vm.CreateOpts, + providerOptsContainer vm.ProviderOptionsContainer, ) error { providerCount := len(opts.VMProviders) if providerCount == 0 { @@ -265,7 +269,7 @@ func CreateCluster( } return vm.ProvidersParallel(opts.VMProviders, func(p vm.Provider) error { - return p.Create(vmLocations[p.Name()], opts, providerOptsContainer[p.Name()]) + return p.Create(l, vmLocations[p.Name()], opts, providerOptsContainer[p.Name()]) }) } diff --git a/pkg/roachprod/cloud/gc.go b/pkg/roachprod/cloud/gc.go index b03b583d1a0f..1c8a5c6a324f 100644 --- a/pkg/roachprod/cloud/gc.go +++ b/pkg/roachprod/cloud/gc.go @@ -24,6 +24,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -120,9 +121,11 @@ func findUserChannel(client *slack.Client, email string) (string, error) { return u.ID, nil } -func postStatus(client *slack.Client, channel string, dryrun bool, s *status, badVMs vm.List) { +func postStatus( + l *logger.Logger, client *slack.Client, channel string, dryrun bool, s *status, badVMs vm.List, +) { if dryrun { - tw := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) + tw := tabwriter.NewWriter(l.Stdout, 0, 8, 2, ' ', 0) for _, c := range s.good { fmt.Fprintf(tw, "good:\t%s\t%s\t(%s)\n", c.Name, c.GCAt().Format(time.Stamp), @@ -276,7 +279,7 @@ func shouldSend(channel string, status *status) (bool, error) { // GCClusters checks all cluster to see if they should be deleted. It only // fails on failure to perform cloud actions. All other actions (load/save // file, email) do not abort. -func GCClusters(cloud *Cloud, dryrun bool) error { +func GCClusters(l *logger.Logger, cloud *Cloud, dryrun bool) error { now := timeutil.Now() var names []string @@ -312,7 +315,7 @@ func GCClusters(cloud *Cloud, dryrun bool) error { // Send out notification to #roachprod-status. client := makeSlackClient() channel, _ := findChannel(client, "roachprod-status", "") - postStatus(client, channel, dryrun, &s, badVMs) + postStatus(l, client, channel, dryrun, &s, badVMs) // Send out user notifications if any of the user's clusters are expired or // will be destroyed. @@ -320,7 +323,7 @@ func GCClusters(cloud *Cloud, dryrun bool) error { if len(status.warn) > 0 || len(status.destroy) > 0 { userChannel, err := findUserChannel(client, user+config.EmailDomain) if err == nil { - postStatus(client, userChannel, dryrun, status, nil) + postStatus(l, client, userChannel, dryrun, status, nil) } else if !errors.Is(err, errNoSlackClient) { log.Infof(context.Background(), "could not deliver Slack DM to %s: %v", user+config.EmailDomain, err) } diff --git a/pkg/roachprod/install/BUILD.bazel b/pkg/roachprod/install/BUILD.bazel index 05836193ea67..81234ac752b5 100644 --- a/pkg/roachprod/install/BUILD.bazel +++ b/pkg/roachprod/install/BUILD.bazel @@ -25,6 +25,7 @@ go_library( "//pkg/roachprod/cloud", "//pkg/roachprod/config", "//pkg/roachprod/errors", + "//pkg/roachprod/logger", "//pkg/roachprod/ssh", "//pkg/roachprod/ui", "//pkg/roachprod/vm/aws", diff --git a/pkg/roachprod/install/cluster_synced.go b/pkg/roachprod/install/cluster_synced.go index 6456d1ddd05e..9994af5f6a58 100644 --- a/pkg/roachprod/install/cluster_synced.go +++ b/pkg/roachprod/install/cluster_synced.go @@ -35,6 +35,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachprod/cloud" "github.com/cockroachdb/cockroach/pkg/roachprod/config" rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/ssh" "github.com/cockroachdb/cockroach/pkg/roachprod/ui" "github.com/cockroachdb/cockroach/pkg/roachprod/vm/aws" @@ -222,12 +223,12 @@ func (c *SyncedCluster) newSession(node Node) (session, error) { // // When running roachprod stop without other flags, the signal is 9 (SIGKILL) // and wait is true. -func (c *SyncedCluster) Stop(ctx context.Context, sig int, wait bool) error { +func (c *SyncedCluster) Stop(ctx context.Context, l *logger.Logger, sig int, wait bool) error { display := fmt.Sprintf("%s: stopping", c.Name) if wait { display += " and waiting" } - return c.Parallel(display, len(c.Nodes), 0, func(i int) ([]byte, error) { + return c.Parallel(l, display, len(c.Nodes), 0, func(i int) ([]byte, error) { sess, err := c.newSession(c.Nodes[i]) if err != nil { return nil, err @@ -274,12 +275,12 @@ fi`, } // Wipe TODO(peter): document -func (c *SyncedCluster) Wipe(ctx context.Context, preserveCerts bool) error { +func (c *SyncedCluster) Wipe(ctx context.Context, l *logger.Logger, preserveCerts bool) error { display := fmt.Sprintf("%s: wiping", c.Name) - if err := c.Stop(ctx, 9, true /* wait */); err != nil { + if err := c.Stop(ctx, l, 9, true /* wait */); err != nil { return err } - return c.Parallel(display, len(c.Nodes), 0, func(i int) ([]byte, error) { + return c.Parallel(l, display, len(c.Nodes), 0, func(i int) ([]byte, error) { sess, err := c.newSession(c.Nodes[i]) if err != nil { return nil, err @@ -310,10 +311,10 @@ sudo rm -fr logs && } // Status TODO(peter): document -func (c *SyncedCluster) Status(ctx context.Context) error { +func (c *SyncedCluster) Status(ctx context.Context, l *logger.Logger) error { display := fmt.Sprintf("%s: status", c.Name) results := make([]string, len(c.Nodes)) - if err := c.Parallel(display, len(c.Nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, display, len(c.Nodes), 0, func(i int) ([]byte, error) { sess, err := c.newSession(c.Nodes[i]) if err != nil { results[i] = err.Error() @@ -349,7 +350,7 @@ fi return err } for i, r := range results { - fmt.Printf(" %2d: %s\n", c.Nodes[i], r) + l.Printf(" %2d: %s\n", c.Nodes[i], r) } return nil } @@ -548,25 +549,6 @@ type RunResultDetails struct { RemoteExitStatus string } -// OutputRunResults prints stdout and stderr of a given RunResultDetails. -func OutputRunResults(results []RunResultDetails) { - for _, result := range results { - if !config.Quiet { - // print STDOUT/STDERR and return if the node returned an err - fmt.Printf("--------------------------\nNODE #%d\n--------------------------\n", result.Node) - fmt.Printf("EXIT STATUS: %s\n", result.RemoteExitStatus) - if result.Stdout != "" { - fmt.Println(result.Stdout) - fmt.Println("End of STDOUT") - } - if result.Stderr != "" { - fmt.Println(result.Stderr) - fmt.Println("End of STDERR") - } - } - } -} - func processStdout(stdout string) (string, string) { retStdout := stdout exitStatusPattern := "LAST EXIT STATUS: " @@ -583,7 +565,7 @@ func processStdout(stdout string) (string, string) { } func runCmdOnSingleNode( - ctx context.Context, c *SyncedCluster, node Node, cmd string, + ctx context.Context, l *logger.Logger, c *SyncedCluster, node Node, cmd string, ) (RunResultDetails, error) { result := RunResultDetails{Node: node} sess, err := c.newSession(node) @@ -601,7 +583,7 @@ func runCmdOnSingleNode( e := expander{ node: node, } - expandedCmd, err := e.expand(ctx, c, cmd) + expandedCmd, err := e.expand(ctx, l, c, cmd) if err != nil { return result, err } @@ -659,7 +641,7 @@ func (e *NonZeroExitCode) Error() string { // title: A description of the command being run that is output to the logs. // cmd: The command to run. func (c *SyncedCluster) Run( - ctx context.Context, stdout, stderr io.Writer, nodes Nodes, title, cmd string, + ctx context.Context, l *logger.Logger, stdout, stderr io.Writer, nodes Nodes, title, cmd string, ) error { // Stream output if we're running the command on only 1 node. stream := len(nodes) == 1 @@ -670,7 +652,7 @@ func (c *SyncedCluster) Run( errs := make([]error, len(nodes)) results := make([]string, len(nodes)) - if err := c.Parallel(display, len(nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, display, len(nodes), 0, func(i int) ([]byte, error) { sess, err := c.newSession(nodes[i]) if err != nil { errs[i] = err @@ -683,7 +665,7 @@ func (c *SyncedCluster) Run( e := expander{ node: nodes[i], } - expandedCmd, err := e.expand(ctx, c, cmd) + expandedCmd, err := e.expand(ctx, l, c, cmd) if err != nil { return nil, err } @@ -740,13 +722,13 @@ func (c *SyncedCluster) Run( // RunWithDetails runs a command on the specified nodes and returns results details and an error. func (c *SyncedCluster) RunWithDetails( - ctx context.Context, nodes Nodes, title, cmd string, + ctx context.Context, l *logger.Logger, nodes Nodes, title, cmd string, ) ([]RunResultDetails, error) { display := fmt.Sprintf("%s: %s", c.Name, title) results := make([]RunResultDetails, len(nodes)) - failed, err := c.ParallelE(display, len(nodes), 0, func(i int) ([]byte, error) { - result, err := runCmdOnSingleNode(ctx, c, nodes[i], cmd) + failed, err := c.ParallelE(l, display, len(nodes), 0, func(i int) ([]byte, error) { + result, err := runCmdOnSingleNode(ctx, l, c, nodes[i], cmd) if err != nil { return nil, err } @@ -762,10 +744,10 @@ func (c *SyncedCluster) RunWithDetails( } // Wait TODO(peter): document -func (c *SyncedCluster) Wait(ctx context.Context) error { +func (c *SyncedCluster) Wait(ctx context.Context, l *logger.Logger) error { display := fmt.Sprintf("%s: waiting for nodes to start", c.Name) errs := make([]error, len(c.Nodes)) - if err := c.Parallel(display, len(c.Nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, display, len(c.Nodes), 0, func(i int) ([]byte, error) { for j := 0; j < 600; j++ { sess, err := c.newSession(c.Nodes[i]) if err != nil { @@ -790,7 +772,7 @@ func (c *SyncedCluster) Wait(ctx context.Context) error { var foundErr bool for i, err := range errs { if err != nil { - fmt.Printf(" %2d: %v\n", c.Nodes[i], err) + l.Printf(" %2d: %v", c.Nodes[i], err) foundErr = true } } @@ -815,7 +797,7 @@ func (c *SyncedCluster) Wait(ctx context.Context) error { // files on all hosts. // // This call strives to be idempotent. -func (c *SyncedCluster) SetupSSH(ctx context.Context) error { +func (c *SyncedCluster) SetupSSH(ctx context.Context, l *logger.Logger) error { if c.IsLocal() { return nil } @@ -828,7 +810,7 @@ func (c *SyncedCluster) SetupSSH(ctx context.Context) error { // Generate an ssh key that we'll distribute to all of the nodes in the // cluster in order to allow inter-node ssh. var sshTar []byte - if err := c.Parallel("generating ssh key", 1, 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "generating ssh key", 1, 0, func(i int) ([]byte, error) { sess, err := c.newSession(1) if err != nil { return nil, err @@ -861,7 +843,7 @@ tar cf - .ssh/id_rsa .ssh/id_rsa.pub .ssh/authorized_keys // Skip the first node which is where we generated the key. nodes := c.Nodes[1:] - if err := c.Parallel("distributing ssh key", len(nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "distributing ssh key", len(nodes), 0, func(i int) ([]byte, error) { sess, err := c.newSession(nodes[i]) if err != nil { return nil, err @@ -883,7 +865,7 @@ tar cf - .ssh/id_rsa .ssh/id_rsa.pub .ssh/authorized_keys // known hosts file in unhashed format, working around a limitation of jsch // (which is used in jepsen tests). ips := make([]string, len(c.Nodes), len(c.Nodes)*2) - if err := c.Parallel("retrieving hosts", len(c.Nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "retrieving hosts", len(c.Nodes), 0, func(i int) ([]byte, error) { for j := 0; j < 20 && ips[i] == ""; j++ { var err error ips[i], err = c.GetInternalIP(ctx, c.Nodes[i]) @@ -904,7 +886,7 @@ tar cf - .ssh/id_rsa .ssh/id_rsa.pub .ssh/authorized_keys ips = append(ips, c.Host(i)) } var knownHostsData []byte - if err := c.Parallel("scanning hosts", 1, 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "scanning hosts", 1, 0, func(i int) ([]byte, error) { sess, err := c.newSession(c.Nodes[i]) if err != nil { return nil, err @@ -946,7 +928,7 @@ exit 1 return err } - if err := c.Parallel("distributing known_hosts", len(c.Nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "distributing known_hosts", len(c.Nodes), 0, func(i int) ([]byte, error) { sess, err := c.newSession(c.Nodes[i]) if err != nil { return nil, err @@ -995,7 +977,7 @@ fi // additional authorized_keys to both the current user (your username on // gce and the shared user on aws) as well as to the shared user on both // platforms. - if err := c.Parallel("adding additional authorized keys", len(c.Nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "adding additional authorized keys", len(c.Nodes), 0, func(i int) ([]byte, error) { sess, err := c.newSession(c.Nodes[i]) if err != nil { return nil, err @@ -1039,7 +1021,7 @@ fi // DistributeCerts will generate and distribute certificates to all of the // nodes. -func (c *SyncedCluster) DistributeCerts(ctx context.Context) error { +func (c *SyncedCluster) DistributeCerts(ctx context.Context, l *logger.Logger) error { dir := "" if c.IsLocal() { dir = c.localVMDir(1) @@ -1048,7 +1030,7 @@ func (c *SyncedCluster) DistributeCerts(ctx context.Context) error { // Check to see if the certs have already been initialized. var existsErr error display := fmt.Sprintf("%s: checking certs", c.Name) - if err := c.Parallel(display, 1, 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, display, 1, 0, func(i int) ([]byte, error) { sess, err := c.newSession(1) if err != nil { return nil, err @@ -1072,7 +1054,7 @@ func (c *SyncedCluster) DistributeCerts(ctx context.Context) error { var ips []string if !c.IsLocal() { ips = make([]string, len(nodes)) - if err := c.Parallel("", len(nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "", len(nodes), 0, func(i int) ([]byte, error) { var err error ips[i], err = c.GetInternalIP(ctx, nodes[i]) return nil, errors.Wrapf(err, "IPs") @@ -1082,7 +1064,7 @@ func (c *SyncedCluster) DistributeCerts(ctx context.Context) error { } // Generate the ca, client and node certificates on the first node. - if err := c.Parallel(display, 1, 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, display, 1, 0, func(i int) ([]byte, error) { sess, err := c.newSession(1) if err != nil { return nil, err @@ -1173,7 +1155,7 @@ tar cvf certs.tar certs // Skip the first node which is where we generated the certs. display = c.Name + ": distributing certs" nodes = nodes[1:] - return c.Parallel(display, len(nodes), 0, func(i int) ([]byte, error) { + return c.Parallel(l, display, len(nodes), 0, func(i int) ([]byte, error) { sess, err := c.newSession(nodes[i]) if err != nil { return nil, err @@ -1208,7 +1190,7 @@ func formatProgress(p float64) string { } // Put TODO(peter): document -func (c *SyncedCluster) Put(ctx context.Context, src, dest string) error { +func (c *SyncedCluster) Put(ctx context.Context, l *logger.Logger, src, dest string) error { // Check if source file exists and if it's a symlink. var potentialSymlinkPath string var err error @@ -1223,8 +1205,8 @@ func (c *SyncedCluster) Put(ctx context.Context, src, dest string) error { return err } redColor, resetColor := "\033[31m", "\033[0m" - fmt.Println(redColor + "WARNING: Source file is a symlink.") - fmt.Printf("WARNING: Remote file will inherit the target permissions '%v'.\n"+resetColor, symlinkTargetInfo.Mode()) + l.Printf(redColor + "WARNING: Source file is a symlink." + resetColor) + l.Printf(redColor+"WARNING: Remote file will inherit the target permissions '%v'."+resetColor, symlinkTargetInfo.Mode()) } // NB: This value was determined with a few experiments. Higher values were @@ -1239,7 +1221,7 @@ func (c *SyncedCluster) Put(ctx context.Context, src, dest string) error { detail = " (scp)" } } - fmt.Printf("%s: putting%s %s %s\n", c.Name, detail, src, dest) + l.Printf("%s: putting%s %s %s\n", c.Name, detail, src, dest) type result struct { index int @@ -1284,7 +1266,7 @@ func (c *SyncedCluster) Put(ctx context.Context, src, dest string) error { e := expander{ node: c.Nodes[i], } - dest, err := e.expand(ctx, c, dest) + dest, err := e.expand(ctx, l, c, dest) if err != nil { return "", err } @@ -1302,7 +1284,7 @@ func (c *SyncedCluster) Put(ctx context.Context, src, dest string) error { node: c.Nodes[i], } var err error - dest, err = e.expand(ctx, c, dest) + dest, err = e.expand(ctx, l, c, dest) if err != nil { results <- result{i, err} return @@ -1403,7 +1385,7 @@ func (c *SyncedCluster) Put(ctx context.Context, src, dest string) error { for done := false; !done; { select { case <-ticker.C: - if config.Quiet { + if config.Quiet && l.File == nil { fmt.Printf(".") } case r, ok := <-results: @@ -1431,16 +1413,16 @@ func (c *SyncedCluster) Put(ctx context.Context, src, dest string) error { fmt.Fprintf(&writer, "\n") } linesMu.Unlock() - _ = writer.Flush(os.Stdout) + _ = writer.Flush(l.Stdout) spinnerIdx++ } } - if config.Quiet { - fmt.Printf("\n") + if config.Quiet && l.File != nil { + l.Printf("\n") linesMu.Lock() for i := range lines { - fmt.Printf(" %2d: %s\n", c.Nodes[i], lines[i]) + l.Printf(" %2d: %s", c.Nodes[i], lines[i]) } linesMu.Unlock() } @@ -1599,14 +1581,14 @@ func (c *SyncedCluster) Logs( } // Get TODO(peter): document -func (c *SyncedCluster) Get(src, dest string) error { +func (c *SyncedCluster) Get(l *logger.Logger, src, dest string) error { // TODO(peter): Only get 10 nodes at a time. When a node completes, output a // line indicating that. var detail string if !c.IsLocal() { detail = " (scp)" } - fmt.Printf("%s: getting%s %s %s\n", c.Name, detail, src, dest) + l.Printf("%s: getting%s %s %s\n", c.Name, detail, src, dest) type result struct { index int @@ -1758,7 +1740,7 @@ func (c *SyncedCluster) Get(src, dest string) error { for done := false; !done; { select { case <-ticker.C: - if config.Quiet { + if config.Quiet && l.File == nil { fmt.Printf(".") } case r, ok := <-results: @@ -1774,7 +1756,7 @@ func (c *SyncedCluster) Get(src, dest string) error { linesMu.Unlock() } } - if !config.Quiet { + if !config.Quiet && l.File == nil { linesMu.Lock() for i := range lines { fmt.Fprintf(&writer, " %2d: ", c.Nodes[i]) @@ -1786,16 +1768,16 @@ func (c *SyncedCluster) Get(src, dest string) error { fmt.Fprintf(&writer, "\n") } linesMu.Unlock() - _ = writer.Flush(os.Stdout) + _ = writer.Flush(l.Stdout) spinnerIdx++ } } - if config.Quiet { - fmt.Printf("\n") + if config.Quiet && l.File == nil { + l.Printf("\n") linesMu.Lock() for i := range lines { - fmt.Printf(" %2d: %s\n", c.Nodes[i], lines[i]) + l.Printf(" %2d: %s", c.Nodes[i], lines[i]) } linesMu.Unlock() } @@ -1807,8 +1789,10 @@ func (c *SyncedCluster) Get(src, dest string) error { } // pgurls returns a map of PG URLs for the given nodes. -func (c *SyncedCluster) pgurls(ctx context.Context, nodes Nodes) (map[Node]string, error) { - hosts, err := c.pghosts(ctx, nodes) +func (c *SyncedCluster) pgurls( + ctx context.Context, l *logger.Logger, nodes Nodes, +) (map[Node]string, error) { + hosts, err := c.pghosts(ctx, l, nodes) if err != nil { return nil, err } @@ -1820,9 +1804,11 @@ func (c *SyncedCluster) pgurls(ctx context.Context, nodes Nodes) (map[Node]strin } // pghosts returns a map of IP addresses for the given nodes. -func (c *SyncedCluster) pghosts(ctx context.Context, nodes Nodes) (map[Node]string, error) { +func (c *SyncedCluster) pghosts( + ctx context.Context, l *logger.Logger, nodes Nodes, +) (map[Node]string, error) { ips := make([]string, len(nodes)) - if err := c.Parallel("", len(nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "", len(nodes), 0, func(i int) ([]byte, error) { var err error ips[i], err = c.GetInternalIP(ctx, nodes[i]) return nil, errors.Wrapf(err, "pghosts") @@ -1838,7 +1824,7 @@ func (c *SyncedCluster) pghosts(ctx context.Context, nodes Nodes) (map[Node]stri } // SSH TODO(peter): document -func (c *SyncedCluster) SSH(ctx context.Context, sshArgs, args []string) error { +func (c *SyncedCluster) SSH(ctx context.Context, l *logger.Logger, sshArgs, args []string) error { if len(c.Nodes) != 1 && len(args) == 0 { // If trying to ssh to more than 1 node and the ssh session is interactive, // try sshing with an iTerm2 split screen configuration. @@ -1854,7 +1840,7 @@ func (c *SyncedCluster) SSH(ctx context.Context, sshArgs, args []string) error { } var expandedArgs []string for _, arg := range args { - expandedArg, err := e.expand(ctx, c, arg) + expandedArg, err := e.expand(ctx, l, c, arg) if err != nil { return err } @@ -1928,13 +1914,13 @@ type ParallelResult struct { // // See ParallelE for more information. func (c *SyncedCluster) Parallel( - display string, count, concurrency int, fn func(i int) ([]byte, error), + l *logger.Logger, display string, count, concurrency int, fn func(i int) ([]byte, error), ) error { - failed, err := c.ParallelE(display, count, concurrency, fn) + failed, err := c.ParallelE(l, display, count, concurrency, fn) if err != nil { sort.Slice(failed, func(i, j int) bool { return failed[i].Index < failed[j].Index }) for _, f := range failed { - fmt.Fprintf(os.Stderr, "%d: %+v: %s\n", f.Index, f.Err, f.Out) + fmt.Fprintf(l.Stderr, "%d: %+v: %s\n", f.Index, f.Err, f.Out) } return err } @@ -1952,7 +1938,7 @@ func (c *SyncedCluster) Parallel( // If err is non-nil, the slice of ParallelResults will contain the // results from any of the failed invocations. func (c *SyncedCluster) ParallelE( - display string, count, concurrency int, fn func(i int) ([]byte, error), + l *logger.Logger, display string, count, concurrency int, fn func(i int) ([]byte, error), ) ([]ParallelResult, error) { if concurrency == 0 || concurrency > count { concurrency = count @@ -1985,7 +1971,7 @@ func (c *SyncedCluster) ParallelE( }() var writer ui.Writer - out := io.Writer(os.Stdout) + out := l.Stdout if display == "" { out = ioutil.Discard } @@ -2007,7 +1993,7 @@ func (c *SyncedCluster) ParallelE( for done := false; !done; { select { case <-ticker.C: - if config.Quiet { + if config.Quiet && l.File == nil { fmt.Fprintf(out, ".") } case r, ok := <-results: @@ -2023,7 +2009,7 @@ func (c *SyncedCluster) ParallelE( } } - if !config.Quiet { + if !config.Quiet && l.File == nil { fmt.Fprint(&writer, display) var n int for i := range complete { @@ -2041,7 +2027,7 @@ func (c *SyncedCluster) ParallelE( } } - if config.Quiet { + if config.Quiet && l.File == nil { fmt.Fprintf(out, "\n") } @@ -2055,27 +2041,27 @@ func (c *SyncedCluster) ParallelE( // to maintain parity with auto-init behavior of `roachprod start` (when // --skip-init) is not specified. The implementation should be kept in // sync with Start(). -func (c *SyncedCluster) Init(ctx context.Context) error { +func (c *SyncedCluster) Init(ctx context.Context, l *logger.Logger) error { // See Start(). We reserve a few special operations for the first node, so we // strive to maintain the same here for interoperability. const firstNodeIdx = 0 - fmt.Printf("%s: initializing cluster\n", c.Name) + l.Printf("%s: initializing cluster\n", c.Name) initOut, err := c.initializeCluster(ctx, firstNodeIdx) if err != nil { return errors.WithDetail(err, "install.Init() failed: unable to initialize cluster.") } if initOut != "" { - fmt.Println(initOut) + l.Printf(initOut) } - fmt.Printf("%s: setting cluster settings\n", c.Name) - clusterSettingsOut, err := c.setClusterSettings(ctx, firstNodeIdx) + l.Printf("%s: setting cluster settings", c.Name) + clusterSettingsOut, err := c.setClusterSettings(ctx, l, firstNodeIdx) if err != nil { return errors.WithDetail(err, "install.Init() failed: unable to set cluster settings.") } if clusterSettingsOut != "" { - fmt.Println(clusterSettingsOut) + l.Printf(clusterSettingsOut) } return nil } diff --git a/pkg/roachprod/install/cockroach.go b/pkg/roachprod/install/cockroach.go index e3059d6cb904..db4e8da7082a 100644 --- a/pkg/roachprod/install/cockroach.go +++ b/pkg/roachprod/install/cockroach.go @@ -24,6 +24,7 @@ import ( "github.com/alessio/shellescape" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/ssh" "github.com/cockroachdb/cockroach/pkg/util/version" "github.com/cockroachdb/errors" @@ -145,11 +146,11 @@ func (st StartTarget) String() string { // (node 1, as understood by SyncedCluster.TargetNodes), we use // `start-single-node` (this was written to provide a short hand to start a // single node cluster with a replication factor of one). -func (c *SyncedCluster) Start(ctx context.Context, startOpts StartOpts) error { +func (c *SyncedCluster) Start(ctx context.Context, l *logger.Logger, startOpts StartOpts) error { if startOpts.Target == StartTenantProxy { return fmt.Errorf("start tenant proxy not implemented") } - if err := c.distributeCerts(ctx); err != nil { + if err := c.distributeCerts(ctx, l); err != nil { return err } @@ -159,8 +160,8 @@ func (c *SyncedCluster) Start(ctx context.Context, startOpts StartOpts) error { parallelism = 1 } - fmt.Printf("%s: starting nodes\n", c.Name) - return c.Parallel("", len(nodes), parallelism, func(nodeIdx int) ([]byte, error) { + l.Printf("%s: starting nodes", c.Name) + return c.Parallel(l, "", len(nodes), parallelism, func(nodeIdx int) ([]byte, error) { node := nodes[nodeIdx] vers, err := getCockroachVersion(ctx, c, node) if err != nil { @@ -169,7 +170,7 @@ func (c *SyncedCluster) Start(ctx context.Context, startOpts StartOpts) error { // NB: if cockroach started successfully, we ignore the output as it is // some harmless start messaging. - if _, err := c.startNode(ctx, node, startOpts, vers); err != nil { + if _, err := c.startNode(ctx, l, node, startOpts, vers); err != nil { return nil, err } @@ -196,27 +197,27 @@ func (c *SyncedCluster) Start(ctx context.Context, startOpts StartOpts) error { shouldInit := !c.useStartSingleNode() if shouldInit { - fmt.Printf("%s: initializing cluster\n", c.Name) + l.Printf("%s: initializing cluster", c.Name) initOut, err := c.initializeCluster(ctx, node) if err != nil { return nil, errors.WithDetail(err, "unable to initialize cluster") } if initOut != "" { - fmt.Println(initOut) + l.Printf(initOut) } } // We're sure to set cluster settings after having initialized the // cluster. - fmt.Printf("%s: setting cluster settings\n", c.Name) - clusterSettingsOut, err := c.setClusterSettings(ctx, node) + l.Printf("%s: setting cluster settings", c.Name) + clusterSettingsOut, err := c.setClusterSettings(ctx, l, node) if err != nil { return nil, errors.Wrap(err, "unable to set cluster settings") } if clusterSettingsOut != "" { - fmt.Println(clusterSettingsOut) + l.Printf(clusterSettingsOut) } return nil, nil }) @@ -285,7 +286,7 @@ func (c *SyncedCluster) NodeUIPort(node Node) int { // // In non-interactive mode, a command specified via the `-e` flag is run against // all nodes. -func (c *SyncedCluster) SQL(ctx context.Context, args []string) error { +func (c *SyncedCluster) SQL(ctx context.Context, l *logger.Logger, args []string) error { if len(args) == 0 || len(c.Nodes) == 1 { // If no arguments, we're going to get an interactive SQL shell. Require // exactly one target and ask SSH to provide a pseudoterminal. @@ -296,17 +297,17 @@ func (c *SyncedCluster) SQL(ctx context.Context, args []string) error { binary := cockroachNodeBinary(c, c.Nodes[0]) allArgs := []string{binary, "sql", "--url", url} allArgs = append(allArgs, ssh.Escape(args)) - return c.SSH(ctx, []string{"-t"}, allArgs) + return c.SSH(ctx, l, []string{"-t"}, allArgs) } // Otherwise, assume the user provided the "-e" flag, so we can reasonably // execute the query on all specified nodes. - return c.RunSQL(ctx, args) + return c.RunSQL(ctx, l, args) } // RunSQL runs a `cockroach sql` command. // It is assumed that the args include the -e flag. -func (c *SyncedCluster) RunSQL(ctx context.Context, args []string) error { +func (c *SyncedCluster) RunSQL(ctx context.Context, l *logger.Logger, args []string) error { type result struct { node Node output string @@ -314,7 +315,7 @@ func (c *SyncedCluster) RunSQL(ctx context.Context, args []string) error { resultChan := make(chan result, len(c.Nodes)) display := fmt.Sprintf("%s: executing sql", c.Name) - if err := c.Parallel(display, len(c.Nodes), 0, func(nodeIdx int) ([]byte, error) { + if err := c.Parallel(l, display, len(c.Nodes), 0, func(nodeIdx int) ([]byte, error) { node := c.Nodes[nodeIdx] sess, err := c.newSession(node) if err != nil { @@ -349,16 +350,16 @@ func (c *SyncedCluster) RunSQL(ctx context.Context, args []string) error { return results[i].node < results[j].node }) for _, r := range results { - fmt.Printf("node %d:\n%s", r.node, r.output) + l.Printf("node %d:\n%s", r.node, r.output) } return nil } func (c *SyncedCluster) startNode( - ctx context.Context, node Node, startOpts StartOpts, vers *version.Version, + ctx context.Context, l *logger.Logger, node Node, startOpts StartOpts, vers *version.Version, ) (string, error) { - startCmd, err := c.generateStartCmd(ctx, node, startOpts, vers) + startCmd, err := c.generateStartCmd(ctx, l, node, startOpts, vers) if err != nil { return "", err } @@ -404,9 +405,9 @@ func (c *SyncedCluster) startNode( } func (c *SyncedCluster) generateStartCmd( - ctx context.Context, node Node, startOpts StartOpts, vers *version.Version, + ctx context.Context, l *logger.Logger, node Node, startOpts StartOpts, vers *version.Version, ) (string, error) { - args, err := c.generateStartArgs(ctx, node, startOpts, vers) + args, err := c.generateStartArgs(ctx, l, node, startOpts, vers) if err != nil { return "", err } @@ -456,7 +457,7 @@ func execStartTemplate(data startTemplateData) (string, error) { // generateStartArgs generates cockroach binary arguments for starting a node. // The first argument is the command (e.g. "start"). func (c *SyncedCluster) generateStartArgs( - ctx context.Context, node Node, startOpts StartOpts, vers *version.Version, + ctx context.Context, l *logger.Logger, node Node, startOpts StartOpts, vers *version.Version, ) ([]string, error) { var args []string @@ -542,7 +543,7 @@ func (c *SyncedCluster) generateStartArgs( node: node, } for _, arg := range startOpts.ExtraArgs { - expandedArg, err := e.expand(ctx, c, arg) + expandedArg, err := e.expand(ctx, l, c, arg) if err != nil { return nil, err } @@ -633,8 +634,10 @@ func (c *SyncedCluster) initializeCluster(ctx context.Context, node Node) (strin return strings.TrimSpace(string(out)), nil } -func (c *SyncedCluster) setClusterSettings(ctx context.Context, node Node) (string, error) { - clusterSettingCmd := c.generateClusterSettingCmd(node) +func (c *SyncedCluster) setClusterSettings( + ctx context.Context, l *logger.Logger, node Node, +) (string, error) { + clusterSettingCmd := c.generateClusterSettingCmd(l, node) sess, err := c.newSession(node) if err != nil { @@ -649,9 +652,9 @@ func (c *SyncedCluster) setClusterSettings(ctx context.Context, node Node) (stri return strings.TrimSpace(string(out)), nil } -func (c *SyncedCluster) generateClusterSettingCmd(node Node) string { +func (c *SyncedCluster) generateClusterSettingCmd(l *logger.Logger, node Node) string { if config.CockroachDevLicense == "" { - fmt.Printf("%s: COCKROACH_DEV_LICENSE unset: enterprise features will be unavailable\n", + l.Printf("%s: COCKROACH_DEV_LICENSE unset: enterprise features will be unavailable\n", c.Name) } @@ -727,10 +730,10 @@ func (c *SyncedCluster) useStartSingleNode() bool { // distributeCerts distributes certs if it's a secure cluster and we're // starting n1. -func (c *SyncedCluster) distributeCerts(ctx context.Context) error { +func (c *SyncedCluster) distributeCerts(ctx context.Context, l *logger.Logger) error { for _, node := range c.TargetNodes() { if node == 1 && c.Secure { - return c.DistributeCerts(ctx) + return c.DistributeCerts(ctx, l) } } return nil diff --git a/pkg/roachprod/install/download.go b/pkg/roachprod/install/download.go index 504213170eff..961577274aed 100644 --- a/pkg/roachprod/install/download.go +++ b/pkg/roachprod/install/download.go @@ -15,9 +15,10 @@ import ( _ "embed" // required for go:embed "fmt" "net/url" - "os" "path" "path/filepath" + + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" ) const ( @@ -29,7 +30,12 @@ var downloadScript string // Download downloads the remote resource, preferring a GCS cache if available. func Download( - ctx context.Context, c *SyncedCluster, sourceURLStr string, sha string, dest string, + ctx context.Context, + l *logger.Logger, + c *SyncedCluster, + sourceURLStr string, + sha string, + dest string, ) error { // https://example.com/foo/bar.txt sourceURL, err := url.Parse(sourceURLStr) @@ -66,7 +72,7 @@ func Download( sha, dest, ) - if err := c.Run(ctx, os.Stdout, os.Stderr, + if err := c.Run(ctx, l, l.Stdout, l.Stderr, downloadNodes, fmt.Sprintf("downloading %s", basename), downloadCmd, @@ -79,7 +85,7 @@ func Download( if c.IsLocal() && !filepath.IsAbs(dest) { src := filepath.Join(c.localVMDir(downloadNodes[0]), dest) cpCmd := fmt.Sprintf(`cp "%s" "%s"`, src, dest) - return c.Run(ctx, os.Stdout, os.Stderr, c.Nodes[1:], "copying to remaining nodes", cpCmd) + return c.Run(ctx, l, l.Stdout, l.Stderr, c.Nodes[1:], "copying to remaining nodes", cpCmd) } return nil diff --git a/pkg/roachprod/install/expander.go b/pkg/roachprod/install/expander.go index a85229c13afc..775506c6e842 100644 --- a/pkg/roachprod/install/expander.go +++ b/pkg/roachprod/install/expander.go @@ -16,6 +16,7 @@ import ( "regexp" "strings" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/errors" ) @@ -41,10 +42,12 @@ type expander struct { } // expanderFunc is a function which may expand a string with a templated value. -type expanderFunc func(context.Context, *SyncedCluster, string) (expanded string, didExpand bool, err error) +type expanderFunc func(context.Context, *logger.Logger, *SyncedCluster, string) (expanded string, didExpand bool, err error) // expand will expand arg if it contains an expander template. -func (e *expander) expand(ctx context.Context, c *SyncedCluster, arg string) (string, error) { +func (e *expander) expand( + ctx context.Context, l *logger.Logger, c *SyncedCluster, arg string, +) (string, error) { var err error s := parameterRe.ReplaceAllStringFunc(arg, func(s string) string { if err != nil { @@ -60,7 +63,7 @@ func (e *expander) expand(ctx context.Context, c *SyncedCluster, arg string) (st e.maybeExpandCertsDir, } for _, f := range expanders { - v, expanded, fErr := f(ctx, c, s) + v, expanded, fErr := f(ctx, l, c, s) if fErr != nil { err = fErr return "" @@ -108,7 +111,7 @@ func (e *expander) maybeExpandMap( // maybeExpandPgURL is an expanderFunc for {pgurl:} func (e *expander) maybeExpandPgURL( - ctx context.Context, c *SyncedCluster, s string, + ctx context.Context, l *logger.Logger, c *SyncedCluster, s string, ) (string, bool, error) { m := pgURLRe.FindStringSubmatch(s) if m == nil { @@ -117,7 +120,7 @@ func (e *expander) maybeExpandPgURL( if e.pgURLs == nil { var err error - e.pgURLs, err = c.pgurls(ctx, allNodes(len(c.VMs))) + e.pgURLs, err = c.pgurls(ctx, l, allNodes(len(c.VMs))) if err != nil { return "", false, err } @@ -129,7 +132,7 @@ func (e *expander) maybeExpandPgURL( // maybeExpandPgHost is an expanderFunc for {pghost:} func (e *expander) maybeExpandPgHost( - ctx context.Context, c *SyncedCluster, s string, + ctx context.Context, l *logger.Logger, c *SyncedCluster, s string, ) (string, bool, error) { m := pgHostRe.FindStringSubmatch(s) if m == nil { @@ -138,7 +141,7 @@ func (e *expander) maybeExpandPgHost( if e.pgHosts == nil { var err error - e.pgHosts, err = c.pghosts(ctx, allNodes(len(c.VMs))) + e.pgHosts, err = c.pghosts(ctx, l, allNodes(len(c.VMs))) if err != nil { return "", false, err } @@ -150,7 +153,7 @@ func (e *expander) maybeExpandPgHost( // maybeExpandPgURL is an expanderFunc for {pgport:} func (e *expander) maybeExpandPgPort( - ctx context.Context, c *SyncedCluster, s string, + ctx context.Context, l *logger.Logger, c *SyncedCluster, s string, ) (string, bool, error) { m := pgPortRe.FindStringSubmatch(s) if m == nil { @@ -170,7 +173,7 @@ func (e *expander) maybeExpandPgPort( // maybeExpandPgURL is an expanderFunc for {uiport:} func (e *expander) maybeExpandUIPort( - ctx context.Context, c *SyncedCluster, s string, + ctx context.Context, l *logger.Logger, c *SyncedCluster, s string, ) (string, bool, error) { m := uiPortRe.FindStringSubmatch(s) if m == nil { @@ -190,7 +193,7 @@ func (e *expander) maybeExpandUIPort( // maybeExpandStoreDir is an expanderFunc for "{store-dir}" func (e *expander) maybeExpandStoreDir( - ctx context.Context, c *SyncedCluster, s string, + ctx context.Context, l *logger.Logger, c *SyncedCluster, s string, ) (string, bool, error) { if !storeDirRe.MatchString(s) { return s, false, nil @@ -200,7 +203,7 @@ func (e *expander) maybeExpandStoreDir( // maybeExpandLogDir is an expanderFunc for "{log-dir}" func (e *expander) maybeExpandLogDir( - ctx context.Context, c *SyncedCluster, s string, + ctx context.Context, l *logger.Logger, c *SyncedCluster, s string, ) (string, bool, error) { if !logDirRe.MatchString(s) { return s, false, nil @@ -210,7 +213,7 @@ func (e *expander) maybeExpandLogDir( // maybeExpandCertsDir is an expanderFunc for "{certs-dir}" func (e *expander) maybeExpandCertsDir( - ctx context.Context, c *SyncedCluster, s string, + ctx context.Context, l *logger.Logger, c *SyncedCluster, s string, ) (string, bool, error) { if !certsDirRe.MatchString(s) { return s, false, nil diff --git a/pkg/roachprod/install/install.go b/pkg/roachprod/install/install.go index fe11ed7ca62a..88de74a20896 100644 --- a/pkg/roachprod/install/install.go +++ b/pkg/roachprod/install/install.go @@ -15,6 +15,8 @@ import ( "context" "fmt" "sort" + + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" ) var installCmds = map[string]string{ @@ -159,12 +161,12 @@ func SortedCmds() []string { } // Install TODO(peter): document -func Install(ctx context.Context, c *SyncedCluster, args []string) error { +func Install(ctx context.Context, l *logger.Logger, c *SyncedCluster, args []string) error { do := func(title, cmd string) error { var buf bytes.Buffer - err := c.Run(ctx, &buf, &buf, c.Nodes, "installing "+title, cmd) + err := c.Run(ctx, l, &buf, &buf, c.Nodes, "installing "+title, cmd) if err != nil { - fmt.Print(buf.String()) + l.Printf(buf.String()) } return err } diff --git a/pkg/roachprod/install/staging.go b/pkg/roachprod/install/staging.go index fa0ffac47c6e..d7e92cc033e3 100644 --- a/pkg/roachprod/install/staging.go +++ b/pkg/roachprod/install/staging.go @@ -14,10 +14,10 @@ import ( "context" "fmt" "net/url" - "os" "path/filepath" "strings" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/util/httputil" "github.com/cockroachdb/errors" ) @@ -125,6 +125,7 @@ func cockroachReleaseURL(version string, arch string) (*url.URL, error) { // available. func StageApplication( ctx context.Context, + l *logger.Logger, c *SyncedCluster, applicationName string, version string, @@ -139,7 +140,7 @@ func StageApplication( switch applicationName { case "cockroach": sha, err := StageRemoteBinary( - ctx, c, applicationName, "cockroach/cockroach", version, archInfo.DebugArchitecture, destDir, + ctx, l, c, applicationName, "cockroach/cockroach", version, archInfo.DebugArchitecture, destDir, ) if err != nil { return err @@ -149,6 +150,7 @@ func StageApplication( for _, library := range crdbLibraries { if err := StageOptionalRemoteLibrary( ctx, + l, c, library, fmt.Sprintf("cockroach/lib/%s", library), @@ -163,11 +165,11 @@ func StageApplication( return nil case "workload": _, err := StageRemoteBinary( - ctx, c, applicationName, "cockroach/workload", version, "" /* arch */, destDir, + ctx, l, c, applicationName, "cockroach/workload", version, "" /* arch */, destDir, ) return err case "release": - return StageCockroachRelease(ctx, c, version, archInfo.ReleaseArchitecture, destDir) + return StageCockroachRelease(ctx, l, c, version, archInfo.ReleaseArchitecture, destDir) default: return fmt.Errorf("unknown application %s", applicationName) } @@ -225,19 +227,22 @@ func URLsForApplication(application string, version string, os string) ([]*url.U // If no SHA is specified, the latest build of the binary is used instead. // Returns the SHA of the resolved binary. func StageRemoteBinary( - ctx context.Context, c *SyncedCluster, applicationName, urlPathBase, SHA, arch, dir string, + ctx context.Context, + l *logger.Logger, + c *SyncedCluster, + applicationName, urlPathBase, SHA, arch, dir string, ) (string, error) { binURL, err := getEdgeURL(urlPathBase, SHA, arch, "") if err != nil { return "", err } - fmt.Printf("Resolved binary url for %s: %s\n", applicationName, binURL) + l.Printf("Resolved binary url for %s: %s", applicationName, binURL) target := filepath.Join(dir, applicationName) cmdStr := fmt.Sprintf( `curl -sfSL -o "%s" "%s" && chmod 755 %s`, target, binURL, target, ) return shaFromEdgeURL(binURL), c.Run( - ctx, os.Stdout, os.Stderr, c.Nodes, fmt.Sprintf("staging binary (%s)", applicationName), cmdStr, + ctx, l, l.Stdout, l.Stderr, c.Nodes, fmt.Sprintf("staging binary (%s)", applicationName), cmdStr, ) } @@ -246,7 +251,10 @@ func StageRemoteBinary( // If no SHA is specified, the latest build of the library is used instead. // It will not error if the library does not exist on the edge. func StageOptionalRemoteLibrary( - ctx context.Context, c *SyncedCluster, libraryName, urlPathBase, SHA, arch, ext, dir string, + ctx context.Context, + l *logger.Logger, + c *SyncedCluster, + libraryName, urlPathBase, SHA, arch, ext, dir string, ) error { url, err := getEdgeURL(urlPathBase, SHA, arch, ext) if err != nil { @@ -254,7 +262,7 @@ func StageOptionalRemoteLibrary( } libDir := filepath.Join(dir, "lib") target := filepath.Join(libDir, libraryName+ext) - fmt.Printf("Resolved library url for %s: %s\n", libraryName, url) + l.Printf("Resolved library url for %s: %s", libraryName, url) cmdStr := fmt.Sprintf( `mkdir -p "%s" && \ curl -sfSL -o "%s" "%s" 2>/dev/null || echo 'optional library %s not found; continuing...'`, @@ -264,13 +272,15 @@ curl -sfSL -o "%s" "%s" 2>/dev/null || echo 'optional library %s not found; cont libraryName+ext, ) return c.Run( - ctx, os.Stdout, os.Stderr, c.Nodes, fmt.Sprintf("staging library (%s)", libraryName), cmdStr, + ctx, l, l.Stdout, l.Stderr, c.Nodes, fmt.Sprintf("staging library (%s)", libraryName), cmdStr, ) } // StageCockroachRelease downloads an official CockroachDB release binary with // the specified version. -func StageCockroachRelease(ctx context.Context, c *SyncedCluster, version, arch, dir string) error { +func StageCockroachRelease( + ctx context.Context, l *logger.Logger, c *SyncedCluster, version, arch, dir string, +) error { if len(version) == 0 { return fmt.Errorf( "release application cannot be staged without specifying a specific version", @@ -280,7 +290,7 @@ func StageCockroachRelease(ctx context.Context, c *SyncedCluster, version, arch, if err != nil { return err } - fmt.Printf("Resolved release url for cockroach version %s: %s\n", version, binURL) + l.Printf("Resolved release url for cockroach version %s: %s", version, binURL) // This command incantation: // - Creates a temporary directory on the remote machine @@ -297,6 +307,6 @@ if [ -d ${tmpdir}/lib ]; then mv ${tmpdir}/lib/* ${dir}/lib; fi && \ chmod 755 ${dir}/cockroach `, dir, binURL) return c.Run( - ctx, os.Stdout, os.Stderr, c.Nodes, "staging cockroach release binary", cmdStr, + ctx, l, l.Stdout, l.Stderr, c.Nodes, "staging cockroach release binary", cmdStr, ) } diff --git a/pkg/cmd/roachtest/logger/BUILD.bazel b/pkg/roachprod/logger/BUILD.bazel similarity index 74% rename from pkg/cmd/roachtest/logger/BUILD.bazel rename to pkg/roachprod/logger/BUILD.bazel index 254f1eac4c5c..aeb6e5a350de 100644 --- a/pkg/cmd/roachtest/logger/BUILD.bazel +++ b/pkg/roachprod/logger/BUILD.bazel @@ -3,7 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "logger", srcs = ["log.go"], - importpath = "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/logger", + importpath = "github.com/cockroachdb/cockroach/pkg/roachprod/logger", visibility = ["//visibility:public"], deps = [ "//pkg/util/log", diff --git a/pkg/cmd/roachtest/logger/log.go b/pkg/roachprod/logger/log.go similarity index 100% rename from pkg/cmd/roachtest/logger/log.go rename to pkg/roachprod/logger/log.go diff --git a/pkg/roachprod/multitenant.go b/pkg/roachprod/multitenant.go index 2e37757f30d0..41d91dcf171e 100644 --- a/pkg/roachprod/multitenant.go +++ b/pkg/roachprod/multitenant.go @@ -16,6 +16,7 @@ import ( "strings" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/errors" ) @@ -27,18 +28,19 @@ import ( // subsets of nodes are selected (e.g. "local:1,2" and "local:3,4"). func StartTenant( ctx context.Context, + l *logger.Logger, tenantCluster string, hostCluster string, startOpts install.StartOpts, clusterSettingsOpts ...install.ClusterSettingOption, ) error { - tc, err := newCluster(tenantCluster, clusterSettingsOpts...) + tc, err := newCluster(l, tenantCluster, clusterSettingsOpts...) if err != nil { return err } // TODO(radu): do we need separate clusterSettingsOpts for the host cluster? - hc, err := newCluster(hostCluster, clusterSettingsOpts...) + hc, err := newCluster(l, hostCluster, clusterSettingsOpts...) if err != nil { return err } @@ -68,8 +70,8 @@ func StartTenant( // so temporarily restrict the target nodes to 1. saveNodes := hc.Nodes hc.Nodes = hc.Nodes[:1] - fmt.Printf("Creating tenant metadata\n") - if err := hc.RunSQL(ctx, []string{ + l.Printf("Creating tenant metadata") + if err := hc.RunSQL(ctx, l, []string{ `-e`, fmt.Sprintf(createTenantIfNotExistsQuery, startOpts.TenantID), }); err != nil { @@ -82,7 +84,7 @@ func StartTenant( kvAddrs = append(kvAddrs, fmt.Sprintf("%s:%d", hc.Host(node), hc.NodePort(node))) } startOpts.KVAddrs = strings.Join(kvAddrs, ",") - return tc.Start(ctx, startOpts) + return tc.Start(ctx, l, startOpts) } // createTenantIfNotExistsQuery is used to initialize the tenant metadata, if diff --git a/pkg/roachprod/roachprod.go b/pkg/roachprod/roachprod.go index f5f92eeba1af..ab2792461813 100644 --- a/pkg/roachprod/roachprod.go +++ b/pkg/roachprod/roachprod.go @@ -32,6 +32,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachprod/cloud" "github.com/cockroachdb/cockroach/pkg/roachprod/config" "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/roachprod/vm/aws" "github.com/cockroachdb/cockroach/pkg/roachprod/vm/azure" @@ -51,7 +52,7 @@ import ( // our naming pattern of "-". The // username must match one of the vm.Provider account names // or the --username override. -func verifyClusterName(clusterName, username string) error { +func verifyClusterName(l *logger.Logger, clusterName, username string) error { if clusterName == "" { return fmt.Errorf("cluster name cannot be blank") } @@ -135,7 +136,9 @@ func sortedClusters() []string { // newCluster initializes a SyncedCluster for the given cluster name. // // The cluster name can include a node selector (e.g. "foo:1-3"). -func newCluster(name string, opts ...install.ClusterSettingOption) (*install.SyncedCluster, error) { +func newCluster( + l *logger.Logger, name string, opts ...install.ClusterSettingOption, +) (*install.SyncedCluster, error) { clusterSettings := install.MakeClusterSettings(opts...) nodeSelector := "all" { @@ -175,7 +178,7 @@ func newCluster(name string, opts ...install.ClusterSettingOption) (*install.Syn // userClusterNameRegexp returns a regexp that matches all clusters owned by the // current user. -func userClusterNameRegexp() (*regexp.Regexp, error) { +func userClusterNameRegexp(l *logger.Logger) (*regexp.Regexp, error) { // In general, we expect that users will have the same // account name across the services they're using, // but we still want to function even if this is not @@ -199,14 +202,14 @@ func userClusterNameRegexp() (*regexp.Regexp, error) { } // Version returns version/build information. -func Version() string { +func Version(l *logger.Logger) string { info := build.GetInfo() return info.Long() } // CachedClusters iterates over all roachprod clusters from the local cache, in // alphabetical order. -func CachedClusters(fn func(clusterName string, numVMs int)) { +func CachedClusters(l *logger.Logger, fn func(clusterName string, numVMs int)) { for _, name := range sortedClusters() { c, ok := readSyncedClusters(name) if !ok { @@ -221,10 +224,10 @@ func CachedClusters(fn func(clusterName string, numVMs int)) { // protects both the reading and the writing in order to prevent the hazard // caused by concurrent goroutines reading cloud state in a different order // than writing it to disk. -func Sync() (*cloud.Cloud, error) { +func Sync(l *logger.Logger) (*cloud.Cloud, error) { lockFile := os.ExpandEnv("$HOME/.roachprod/LOCK") if !config.Quiet { - fmt.Println("Syncing...") + l.Printf("Syncing...") } // Acquire a filesystem lock so that two concurrent synchronizations of // roachprod state don't clobber each other. @@ -236,7 +239,7 @@ func Sync() (*cloud.Cloud, error) { return nil, errors.Wrap(err, "acquiring lock on %q") } defer f.Close() - cld, err := cloud.ListCloud() + cld, err := cloud.ListCloud(l) if err != nil { return nil, err } @@ -277,14 +280,14 @@ func Sync() (*cloud.Cloud, error) { // clouds. if refreshDNS { if !config.Quiet { - fmt.Println("Refreshing DNS entries...") + l.Printf("Refreshing DNS entries...") } - if err := gce.SyncDNS(vms); err != nil { - fmt.Fprintf(os.Stderr, "failed to update %s DNS: %v", gce.Subdomain, err) + if err := gce.SyncDNS(l, vms); err != nil { + fmt.Fprintf(l.Stderr, "failed to update %s DNS: %v", gce.Subdomain, err) } } else { if !config.Quiet { - fmt.Println("Not refreshing DNS entries. We did not have all the VMs.") + l.Printf("Not refreshing DNS entries. We did not have all the VMs.") } } @@ -300,7 +303,7 @@ func Sync() (*cloud.Cloud, error) { // List returns a cloud.Cloud struct of all roachprod clusters matching clusterNamePattern. // Alternatively, the 'listMine' option can be provided to get the clusters that are owned // by the current user. -func List(listMine bool, clusterNamePattern string) (cloud.Cloud, error) { +func List(l *logger.Logger, listMine bool, clusterNamePattern string) (cloud.Cloud, error) { if err := LoadClusters(); err != nil { return cloud.Cloud{}, err } @@ -308,7 +311,7 @@ func List(listMine bool, clusterNamePattern string) (cloud.Cloud, error) { if clusterNamePattern == "" { if listMine { var err error - listPattern, err = userClusterNameRegexp() + listPattern, err = userClusterNameRegexp(l) if err != nil { return cloud.Cloud{}, err } @@ -324,7 +327,7 @@ func List(listMine bool, clusterNamePattern string) (cloud.Cloud, error) { } } - cld, err := Sync() + cld, err := Sync(l) if err != nil { return cloud.Cloud{}, err } @@ -341,6 +344,7 @@ func List(listMine bool, clusterNamePattern string) (cloud.Cloud, error) { // Run runs a command on the nodes in a cluster. func Run( ctx context.Context, + l *logger.Logger, clusterName, SSHOptions, processTag string, secure bool, stdout, stderr io.Writer, @@ -349,7 +353,7 @@ func Run( if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName, install.SecureOption(secure), install.TagOption(processTag)) + c, err := newCluster(l, clusterName, install.SecureOption(secure), install.TagOption(processTag)) if err != nil { return err } @@ -357,7 +361,7 @@ func Run( // Use "ssh" if an interactive session was requested (i.e. there is no // remote command to run). if len(cmdArray) == 0 { - return c.SSH(ctx, strings.Split(SSHOptions, " "), cmdArray) + return c.SSH(ctx, l, strings.Split(SSHOptions, " "), cmdArray) } cmd := strings.TrimSpace(strings.Join(cmdArray, " ")) @@ -365,17 +369,21 @@ func Run( if len(title) > 30 { title = title[:27] + "..." } - return c.Run(ctx, stdout, stderr, c.Nodes, title, cmd) + return c.Run(ctx, l, stdout, stderr, c.Nodes, title, cmd) } // RunWithDetails runs a command on the nodes in a cluster. func RunWithDetails( - ctx context.Context, clusterName, SSHOptions, processTag string, secure bool, cmdArray []string, + ctx context.Context, + l *logger.Logger, + clusterName, SSHOptions, processTag string, + secure bool, + cmdArray []string, ) ([]install.RunResultDetails, error) { if err := LoadClusters(); err != nil { return nil, err } - c, err := newCluster(clusterName, install.SecureOption(secure), install.TagOption(processTag)) + c, err := newCluster(l, clusterName, install.SecureOption(secure), install.TagOption(processTag)) if err != nil { return nil, err } @@ -383,7 +391,7 @@ func RunWithDetails( // Use "ssh" if an interactive session was requested (i.e. there is no // remote command to run). if len(cmdArray) == 0 { - return nil, c.SSH(ctx, strings.Split(SSHOptions, " "), cmdArray) + return nil, c.SSH(ctx, l, strings.Split(SSHOptions, " "), cmdArray) } cmd := strings.TrimSpace(strings.Join(cmdArray, " ")) @@ -391,27 +399,31 @@ func RunWithDetails( if len(title) > 30 { title = title[:27] + "..." } - return c.RunWithDetails(ctx, c.Nodes, title, cmd) + return c.RunWithDetails(ctx, l, c.Nodes, title, cmd) } // SQL runs `cockroach sql` on a remote cluster. -func SQL(ctx context.Context, clusterName string, secure bool, cmdArray []string) error { +func SQL( + ctx context.Context, l *logger.Logger, clusterName string, secure bool, cmdArray []string, +) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName, install.SecureOption(secure)) + c, err := newCluster(l, clusterName, install.SecureOption(secure)) if err != nil { return err } - return c.SQL(ctx, cmdArray) + return c.SQL(ctx, l, cmdArray) } // IP gets the ip addresses of the nodes in a cluster. -func IP(ctx context.Context, clusterName string, external bool) ([]string, error) { +func IP( + ctx context.Context, l *logger.Logger, clusterName string, external bool, +) ([]string, error) { if err := LoadClusters(); err != nil { return nil, err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return nil, err } @@ -425,7 +437,7 @@ func IP(ctx context.Context, clusterName string, external bool) ([]string, error } } else { var err error - if err := c.Parallel("", len(nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "", len(nodes), 0, func(i int) ([]byte, error) { ips[i], err = c.GetInternalIP(ctx, nodes[i]) return nil, err }); err != nil { @@ -436,26 +448,29 @@ func IP(ctx context.Context, clusterName string, external bool) ([]string, error } // Status retrieves the status of nodes in a cluster. -func Status(ctx context.Context, clusterName, processTag string) error { +func Status(ctx context.Context, l *logger.Logger, clusterName, processTag string) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName, install.TagOption(processTag)) + c, err := newCluster(l, clusterName, install.TagOption(processTag)) if err != nil { return err } - return c.Status(ctx) + return c.Status(ctx, l) } // Stage stages release and edge binaries to the cluster. // stageOS, stageDir, version can be "" to use default values func Stage( - ctx context.Context, clusterName string, stageOS, stageDir, applicationName, version string, + ctx context.Context, + l *logger.Logger, + clusterName string, + stageOS, stageDir, applicationName, version string, ) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } @@ -472,11 +487,11 @@ func Stage( dir = stageDir } - return install.StageApplication(ctx, c, applicationName, version, os, dir) + return install.StageApplication(ctx, l, c, applicationName, version, os, dir) } // Reset resets all VMs in a cluster. -func Reset(clusterName string) error { +func Reset(l *logger.Logger, clusterName string) error { if err := LoadClusters(); err != nil { return err } @@ -485,7 +500,7 @@ func Reset(clusterName string) error { return nil } - cld, err := cloud.ListCloud() + cld, err := cloud.ListCloud(l) if err != nil { return err } @@ -500,11 +515,11 @@ func Reset(clusterName string) error { } // SetupSSH sets up the keys and host keys for the vms in the cluster. -func SetupSSH(ctx context.Context, clusterName string) error { +func SetupSSH(ctx context.Context, l *logger.Logger, clusterName string) error { if err := LoadClusters(); err != nil { return err } - cld, err := Sync() + cld, err := Sync(l) if err != nil { return err } @@ -530,7 +545,7 @@ func SetupSSH(ctx context.Context, clusterName string) error { return err } - cloudCluster.PrintDetails() + cloudCluster.PrintDetails(l) // Run ssh-keygen -R serially on each new VM in case an IP address has been recycled for _, v := range cloudCluster.VMs { cmd := exec.Command("ssh-keygen", "-R", v.PublicIP) @@ -546,7 +561,7 @@ func SetupSSH(ctx context.Context, clusterName string) error { return err } - installCluster, err := newCluster(clusterName) + installCluster, err := newCluster(l, clusterName) if err != nil { return err } @@ -557,7 +572,7 @@ func SetupSSH(ctx context.Context, clusterName string) error { installCluster.VMs[i].RemoteUser = config.OSUser.Username } } - if err := installCluster.Wait(ctx); err != nil { + if err := installCluster.Wait(ctx, l); err != nil { return err } // Fetch public keys from gcloud to set up ssh access for all users into the @@ -566,15 +581,15 @@ func SetupSSH(ctx context.Context, clusterName string) error { if err != nil { return errors.Wrap(err, "failed to retrieve authorized keys from gcloud") } - return installCluster.SetupSSH(ctx) + return installCluster.SetupSSH(ctx, l) } // Extend extends the lifetime of the specified cluster to prevent it from being destroyed. -func Extend(clusterName string, lifetime time.Duration) error { +func Extend(l *logger.Logger, clusterName string, lifetime time.Duration) error { if err := LoadClusters(); err != nil { return err } - cld, err := cloud.ListCloud() + cld, err := cloud.ListCloud(l) if err != nil { return err } @@ -589,7 +604,7 @@ func Extend(clusterName string, lifetime time.Duration) error { } // Reload the clusters and print details. - cld, err = cloud.ListCloud() + cld, err = cloud.ListCloud(l) if err != nil { return err } @@ -599,7 +614,7 @@ func Extend(clusterName string, lifetime time.Duration) error { return fmt.Errorf("cluster %s does not exist", clusterName) } - c.PrintDetails() + c.PrintDetails(l) return nil } @@ -617,6 +632,7 @@ func DefaultStartOpts() install.StartOpts { // Start starts nodes on a cluster. func Start( ctx context.Context, + l *logger.Logger, clusterName string, startOpts install.StartOpts, clusterSettingsOpts ...install.ClusterSettingOption, @@ -624,18 +640,18 @@ func Start( if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName, clusterSettingsOpts...) + c, err := newCluster(l, clusterName, clusterSettingsOpts...) if err != nil { return err } - return c.Start(ctx, startOpts) + return c.Start(ctx, l, startOpts) } // Monitor monitors the status of cockroach nodes in a cluster. func Monitor( - ctx context.Context, clusterName string, opts install.MonitorOpts, + ctx context.Context, l *logger.Logger, clusterName string, opts install.MonitorOpts, ) (chan install.NodeMonitorInfo, error) { - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return nil, err } @@ -659,47 +675,47 @@ func DefaultStopOpts() StopOpts { } // Stop stops nodes on a cluster. -func Stop(ctx context.Context, clusterName string, opts StopOpts) error { +func Stop(ctx context.Context, l *logger.Logger, clusterName string, opts StopOpts) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName, install.TagOption(opts.ProcessTag)) + c, err := newCluster(l, clusterName, install.TagOption(opts.ProcessTag)) if err != nil { return err } - return c.Stop(ctx, opts.Sig, opts.Wait) + return c.Stop(ctx, l, opts.Sig, opts.Wait) } // Init initializes the cluster. -func Init(ctx context.Context, clusterName string) error { +func Init(ctx context.Context, l *logger.Logger, clusterName string) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } - return c.Init(ctx) + return c.Init(ctx, l) } // Wipe wipes the nodes in a cluster. -func Wipe(ctx context.Context, clusterName string, preserveCerts bool) error { +func Wipe(ctx context.Context, l *logger.Logger, clusterName string, preserveCerts bool) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } - return c.Wipe(ctx, preserveCerts) + return c.Wipe(ctx, l, preserveCerts) } // Reformat reformats disks in a cluster to use the specified filesystem. -func Reformat(ctx context.Context, clusterName string, fs string) error { +func Reformat(ctx context.Context, l *logger.Logger, clusterName string, fs string) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } @@ -707,7 +723,7 @@ func Reformat(ctx context.Context, clusterName string, fs string) error { var fsCmd string switch fs { case vm.Zfs: - if err := install.Install(ctx, c, []string{vm.Zfs}); err != nil { + if err := install.Install(ctx, l, c, []string{vm.Zfs}); err != nil { return err } fsCmd = `sudo zpool create -f data1 -m /mnt/data1 /dev/sdb` @@ -717,7 +733,7 @@ func Reformat(ctx context.Context, clusterName string, fs string) error { return fmt.Errorf("unknown filesystem %q", fs) } - err = c.Run(ctx, os.Stdout, os.Stderr, c.Nodes, "reformatting", fmt.Sprintf(` + err = c.Run(ctx, l, os.Stdout, os.Stderr, c.Nodes, "reformatting", fmt.Sprintf(` set -euo pipefail if sudo zpool list -Ho name 2>/dev/null | grep ^data1$; then sudo zpool destroy -f data1 @@ -735,76 +751,80 @@ sudo chmod 777 /mnt/data1 } // Install installs third party software. -func Install(ctx context.Context, clusterName string, software []string) error { +func Install(ctx context.Context, l *logger.Logger, clusterName string, software []string) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } - return install.Install(ctx, c, software) + return install.Install(ctx, l, c, software) } // Download downloads 3rd party tools, using a GCS cache if possible. -func Download(ctx context.Context, clusterName string, src, sha, dest string) error { +func Download( + ctx context.Context, l *logger.Logger, clusterName string, src, sha, dest string, +) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } - return install.Download(ctx, c, src, sha, dest) + return install.Download(ctx, l, c, src, sha, dest) } // DistributeCerts distributes certificates to the nodes in a cluster. // If the certificates already exist, no action is taken. -func DistributeCerts(ctx context.Context, clusterName string) error { +func DistributeCerts(ctx context.Context, l *logger.Logger, clusterName string) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } - return c.DistributeCerts(ctx) + return c.DistributeCerts(ctx, l) } // Put copies a local file to the nodes in a cluster. -func Put(ctx context.Context, clusterName, src, dest string, useTreeDist bool) error { +func Put( + ctx context.Context, l *logger.Logger, clusterName, src, dest string, useTreeDist bool, +) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName, install.UseTreeDistOption(useTreeDist)) + c, err := newCluster(l, clusterName, install.UseTreeDistOption(useTreeDist)) if err != nil { return err } - return c.Put(ctx, src, dest) + return c.Put(ctx, l, src, dest) } // Get copies a remote file from the nodes in a cluster. // If the file is retrieved from multiple nodes the destination // file name will be prefixed with the node number. -func Get(clusterName, src, dest string) error { +func Get(l *logger.Logger, clusterName, src, dest string) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } - return c.Get(src, dest) + return c.Get(l, src, dest) } // PgURL generates pgurls for the nodes in a cluster. func PgURL( - ctx context.Context, clusterName, certsDir string, external, secure bool, + ctx context.Context, l *logger.Logger, clusterName, certsDir string, external, secure bool, ) ([]string, error) { if err := LoadClusters(); err != nil { return nil, err } - c, err := newCluster(clusterName, install.SecureOption(secure), install.PGUrlCertsDirOption(certsDir)) + c, err := newCluster(l, clusterName, install.SecureOption(secure), install.PGUrlCertsDirOption(certsDir)) if err != nil { return nil, err } @@ -817,7 +837,7 @@ func PgURL( } } else { var err error - if err := c.Parallel("", len(nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "", len(nodes), 0, func(i int) ([]byte, error) { ips[i], err = c.GetInternalIP(ctx, nodes[i]) return nil, err }); err != nil { @@ -840,12 +860,12 @@ func PgURL( // AdminURL generates admin UI URLs for the nodes in a cluster. func AdminURL( - clusterName, path string, usePublicIPs, openInBrowser, secure bool, + l *logger.Logger, clusterName, path string, usePublicIPs, openInBrowser, secure bool, ) ([]string, error) { if err := LoadClusters(); err != nil { return nil, err } - c, err := newCluster(clusterName, install.SecureOption(secure)) + c, err := newCluster(l, clusterName, install.SecureOption(secure)) if err != nil { return nil, err } @@ -857,7 +877,7 @@ func AdminURL( // verify DNS is working / fallback to IPs if not. if i == 0 && !usePublicIPs { if _, err := net.LookupHost(host); err != nil { - fmt.Fprintf(os.Stderr, "no valid DNS (yet?). might need to re-run `sync`?\n") + fmt.Fprintf(l.Stderr, "no valid DNS (yet?). might need to re-run `sync`?\n") usePublicIPs = true } } @@ -894,11 +914,11 @@ type PprofOpts struct { } // Pprof TODO -func Pprof(clusterName string, opts PprofOpts) error { +func Pprof(l *logger.Logger, clusterName string, opts PprofOpts) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } @@ -926,7 +946,7 @@ func Pprof(clusterName string, opts PprofOpts) error { httpClient := httputil.NewClientWithTimeout(timeout) startTime := timeutil.Now().Unix() nodes := c.TargetNodes() - failed, err := c.ParallelE(description, len(nodes), 0, func(i int) ([]byte, error) { + failed, err := c.ParallelE(l, description, len(nodes), 0, func(i int) ([]byte, error) { node := nodes[i] host := c.Host(node) port := c.NodeUIPort(node) @@ -944,11 +964,11 @@ func Pprof(clusterName string, opts PprofOpts) error { defer func() { err := file.Close() if err != nil && !errors.Is(err, oserror.ErrClosed) { - fmt.Fprintf(os.Stderr, "warning: could not close temporary file") + fmt.Fprintf(l.Stderr, "warning: could not close temporary file") } err = os.Remove(file.Name()) if err != nil && !oserror.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "warning: could not remove temporary file") + fmt.Fprintf(l.Stderr, "warning: could not remove temporary file") } }() @@ -983,13 +1003,13 @@ func Pprof(clusterName string, opts PprofOpts) error { }) for _, s := range outputFiles { - fmt.Printf("Created %s\n", s) + l.Printf("Created %s", s) } if err != nil { sort.Slice(failed, func(i, j int) bool { return failed[i].Index < failed[j].Index }) for _, f := range failed { - fmt.Fprintf(os.Stderr, "%d: %+v: %s\n", f.Index, f.Err, f.Out) + fmt.Fprintf(l.Stderr, "%d: %+v: %s\n", f.Index, f.Err, f.Out) } exit.WithCode(exit.UnspecifiedError()) } @@ -1018,7 +1038,9 @@ func Pprof(clusterName string, opts PprofOpts) error { } // Destroy TODO -func Destroy(destroyAllMine bool, destroyAllLocal bool, clusterNames ...string) error { +func Destroy( + l *logger.Logger, destroyAllMine bool, destroyAllLocal bool, clusterNames ...string, +) error { if err := LoadClusters(); err != nil { return errors.Wrap(err, "problem loading clusters") } @@ -1034,11 +1056,11 @@ func Destroy(destroyAllMine bool, destroyAllLocal bool, clusterNames ...string) if destroyAllLocal { return errors.New("--all-mine cannot be combined with --all-local") } - destroyPattern, err := userClusterNameRegexp() + destroyPattern, err := userClusterNameRegexp(l) if err != nil { return err } - cld, err = cloud.ListCloud() + cld, err = cloud.ListCloud(l) if err != nil { return err } @@ -1064,45 +1086,45 @@ func Destroy(destroyAllMine bool, destroyAllLocal bool, clusterNames ...string) func(ctx context.Context, idx int) error { name := clusterNames[idx] if config.IsLocalClusterName(name) { - return destroyLocalCluster(ctx, name) + return destroyLocalCluster(ctx, l, name) } if cld == nil { var err error - cld, err = cloud.ListCloud() + cld, err = cloud.ListCloud(l) if err != nil { return err } } - return destroyCluster(cld, name) + return destroyCluster(cld, l, name) }); err != nil { return err } - fmt.Println("OK") + l.Printf("OK") return nil } -func destroyCluster(cld *cloud.Cloud, clusterName string) error { +func destroyCluster(cld *cloud.Cloud, l *logger.Logger, clusterName string) error { c, ok := cld.Clusters[clusterName] if !ok { return fmt.Errorf("cluster %s does not exist", clusterName) } - fmt.Printf("Destroying cluster %s with %d nodes\n", clusterName, len(c.VMs)) + l.Printf("Destroying cluster %s with %d nodes", clusterName, len(c.VMs)) return cloud.DestroyCluster(c) } -func destroyLocalCluster(ctx context.Context, clusterName string) error { +func destroyLocalCluster(ctx context.Context, l *logger.Logger, clusterName string) error { if _, ok := readSyncedClusters(clusterName); !ok { return fmt.Errorf("cluster %s does not exist", clusterName) } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } - if err := c.Wipe(ctx, false); err != nil { + if err := c.Wipe(ctx, l, false); err != nil { return err } - return local.DeleteCluster(clusterName) + return local.DeleteCluster(l, clusterName) } // ClusterAlreadyExistsError is returned when the cluster name passed to Create is already used by another cluster. @@ -1114,8 +1136,8 @@ func (e *ClusterAlreadyExistsError) Error() string { return fmt.Sprintf("cluster %s already exists", e.name) } -func cleanupFailedCreate(clusterName string) error { - cld, err := cloud.ListCloud() +func cleanupFailedCreate(l *logger.Logger, clusterName string) error { + cld, err := cloud.ListCloud(l) if err != nil { return err } @@ -1131,6 +1153,7 @@ func cleanupFailedCreate(clusterName string) error { // Create TODO func Create( ctx context.Context, + l *logger.Logger, username string, numNodes int, createVMOpts vm.CreateOpts, @@ -1144,7 +1167,7 @@ func Create( return fmt.Errorf("number of nodes must be in [1..999]") } clusterName := createVMOpts.ClusterName - if err := verifyClusterName(clusterName, username); err != nil { + if err := verifyClusterName(l, clusterName, username); err != nil { return err } @@ -1155,16 +1178,16 @@ func Create( if errors.HasType(retErr, (*ClusterAlreadyExistsError)(nil)) { return } - fmt.Fprintf(os.Stderr, "Cleaning up partially-created cluster (prev err: %s)\n", retErr) - if err := cleanupFailedCreate(clusterName); err != nil { - fmt.Fprintf(os.Stderr, "Error while cleaning up partially-created cluster: %s\n", err) + fmt.Fprintf(l.Stderr, "Cleaning up partially-created cluster (prev err: %s)\n", retErr) + if err := cleanupFailedCreate(l, clusterName); err != nil { + fmt.Fprintf(l.Stderr, "Error while cleaning up partially-created cluster: %s\n", err) } else { - fmt.Fprintf(os.Stderr, "Cleaning up OK\n") + fmt.Fprintf(l.Stderr, "Cleaning up OK\n") } }() if !config.IsLocalClusterName(clusterName) { - cld, err := cloud.ListCloud() + cld, err := cloud.ListCloud(l) if err != nil { return err } @@ -1190,8 +1213,8 @@ func Create( } } - fmt.Printf("Creating cluster %s with %d nodes\n", clusterName, numNodes) - if createErr := cloud.CreateCluster(numNodes, createVMOpts, providerOptsContainer); createErr != nil { + l.Printf("Creating cluster %s with %d nodes", clusterName, numNodes) + if createErr := cloud.CreateCluster(l, numNodes, createVMOpts, providerOptsContainer); createErr != nil { return createErr } @@ -1199,18 +1222,18 @@ func Create( // No need for ssh for local clusters. return LoadClusters() } - return SetupSSH(ctx, clusterName) + return SetupSSH(ctx, l, clusterName) } // GC garbage-collects expired clusters and unused SSH keypairs in AWS. -func GC(dryrun bool) error { +func GC(l *logger.Logger, dryrun bool) error { if err := LoadClusters(); err != nil { return err } - cld, err := cloud.ListCloud() + cld, err := cloud.ListCloud(l) if err == nil { // GCClusters depends on ListCloud so only call it if ListCloud runs without errors - err = cloud.GCClusters(cld, dryrun) + err = cloud.GCClusters(l, cld, dryrun) } otherErr := cloud.GCAWSKeyPairs(dryrun) return errors.CombineErrors(err, otherErr) @@ -1225,11 +1248,11 @@ type LogsOpts struct { } // Logs TODO -func Logs(clusterName, dest, username string, logsOpts LogsOpts) error { +func Logs(l *logger.Logger, clusterName, dest, username string, logsOpts LogsOpts) error { if err := LoadClusters(); err != nil { return err } - c, err := newCluster(clusterName) + c, err := newCluster(l, clusterName) if err != nil { return err } @@ -1240,7 +1263,7 @@ func Logs(clusterName, dest, username string, logsOpts LogsOpts) error { } // StageURL TODO -func StageURL(applicationName, version, stageOS string) ([]*url.URL, error) { +func StageURL(l *logger.Logger, applicationName, version, stageOS string) ([]*url.URL, error) { os := runtime.GOOS if stageOS != "" { os = stageOS diff --git a/pkg/roachprod/vm/BUILD.bazel b/pkg/roachprod/vm/BUILD.bazel index a13f07a8f0ec..38ef44d8b4f8 100644 --- a/pkg/roachprod/vm/BUILD.bazel +++ b/pkg/roachprod/vm/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/roachprod/config", + "//pkg/roachprod/logger", "@com_github_cockroachdb_errors//:errors", "@com_github_spf13_pflag//:pflag", "@org_golang_x_sync//errgroup", diff --git a/pkg/roachprod/vm/aws/BUILD.bazel b/pkg/roachprod/vm/aws/BUILD.bazel index 0f2a0c913fc5..baba6c4a946b 100644 --- a/pkg/roachprod/vm/aws/BUILD.bazel +++ b/pkg/roachprod/vm/aws/BUILD.bazel @@ -14,6 +14,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/roachprod/config", + "//pkg/roachprod/logger", "//pkg/roachprod/vm", "//pkg/roachprod/vm/flagstub", "//pkg/util/log", diff --git a/pkg/roachprod/vm/aws/aws.go b/pkg/roachprod/vm/aws/aws.go index 70f282518a66..da874925dc42 100644 --- a/pkg/roachprod/vm/aws/aws.go +++ b/pkg/roachprod/vm/aws/aws.go @@ -22,6 +22,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/roachprod/vm/flagstub" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -384,7 +385,7 @@ func (p *Provider) ConfigSSH(zones []string) error { // Create is part of the vm.Provider interface. func (p *Provider) Create( - names []string, opts vm.CreateOpts, vmProviderOpts vm.ProviderOpts, + l *logger.Logger, names []string, opts vm.CreateOpts, vmProviderOpts vm.ProviderOpts, ) error { providerOpts := vmProviderOpts.(*ProviderOpts) expandedZones, err := vm.ExpandZonesFlag(providerOpts.CreateZones) @@ -445,7 +446,7 @@ func (p *Provider) Create( if err := g.Wait(); err != nil { return err } - return p.waitForIPs(names, regions, providerOpts) + return p.waitForIPs(l, names, regions, providerOpts) } // waitForIPs waits until AWS reports both internal and external IP addresses @@ -453,7 +454,9 @@ func (p *Provider) Create( // list the new VMs after the creation might find VMs without an external IP. // We do a bad job at higher layers detecting this lack of IP which can lead to // commands hanging indefinitely. -func (p *Provider) waitForIPs(names []string, regions []string, opts *ProviderOpts) error { +func (p *Provider) waitForIPs( + l *logger.Logger, names []string, regions []string, opts *ProviderOpts, +) error { waitForIPRetry := retry.Start(retry.Options{ InitialBackoff: 100 * time.Millisecond, MaxBackoff: 500 * time.Millisecond, @@ -467,7 +470,7 @@ func (p *Provider) waitForIPs(names []string, regions []string, opts *ProviderOp return m } for waitForIPRetry.Next() { - vms, err := p.listRegions(regions, *opts) + vms, err := p.listRegions(l, regions, *opts) if err != nil { return err } @@ -609,18 +612,20 @@ func (p *Provider) stsGetCallerIdentity() (string, error) { } // List is part of the vm.Provider interface. -func (p *Provider) List() (vm.List, error) { +func (p *Provider) List(l *logger.Logger) (vm.List, error) { regions, err := p.allRegions(p.Config.availabilityZoneNames()) if err != nil { return nil, err } defaultOpts := p.CreateProviderOpts().(*ProviderOpts) - return p.listRegions(regions, *defaultOpts) + return p.listRegions(l, regions, *defaultOpts) } // listRegions lists VMs in the regions passed. // It ignores region-specific errors. -func (p *Provider) listRegions(regions []string, opts ProviderOpts) (vm.List, error) { +func (p *Provider) listRegions( + l *logger.Logger, regions []string, opts ProviderOpts, +) (vm.List, error) { var ret vm.List var mux syncutil.Mutex var g errgroup.Group @@ -631,7 +636,7 @@ func (p *Provider) listRegions(regions []string, opts ProviderOpts) (vm.List, er g.Go(func() error { vms, err := p.listRegion(region, opts) if err != nil { - fmt.Printf("Failed to list AWS VMs in region: %s\n%v\n", region, err) + l.Printf("Failed to list AWS VMs in region: %s\n%v\n", region, err) return nil } mux.Lock() diff --git a/pkg/roachprod/vm/azure/BUILD.bazel b/pkg/roachprod/vm/azure/BUILD.bazel index 524263b0644d..f512f92a1059 100644 --- a/pkg/roachprod/vm/azure/BUILD.bazel +++ b/pkg/roachprod/vm/azure/BUILD.bazel @@ -14,6 +14,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/roachprod/config", + "//pkg/roachprod/logger", "//pkg/roachprod/vm", "//pkg/roachprod/vm/flagstub", "//pkg/util/log", diff --git a/pkg/roachprod/vm/azure/azure.go b/pkg/roachprod/vm/azure/azure.go index 53894ed6d6d2..a2a68168bd0e 100644 --- a/pkg/roachprod/vm/azure/azure.go +++ b/pkg/roachprod/vm/azure/azure.go @@ -28,6 +28,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/roachprod/vm/flagstub" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -123,7 +124,7 @@ func getAzureDefaultLabelMap(opts vm.CreateOpts) map[string]string { // Create implements vm.Provider. func (p *Provider) Create( - names []string, opts vm.CreateOpts, vmProviderOpts vm.ProviderOpts, + l *logger.Logger, names []string, opts vm.CreateOpts, vmProviderOpts vm.ProviderOpts, ) error { providerOpts := vmProviderOpts.(*ProviderOpts) // Load the user's SSH public key to configure the resulting VMs. @@ -394,7 +395,7 @@ func (p *Provider) FindActiveAccount() (string, error) { // List implements the vm.Provider interface. This will query all // Azure VMs in the subscription and select those with a roachprod tag. -func (p *Provider) List() (vm.List, error) { +func (p *Provider) List(l *logger.Logger) (vm.List, error) { ctx, cancel := context.WithTimeout(context.Background(), p.OperationTimeout) defer cancel() diff --git a/pkg/roachprod/vm/flagstub/BUILD.bazel b/pkg/roachprod/vm/flagstub/BUILD.bazel index 7515a75cc8ed..b123b806cd9c 100644 --- a/pkg/roachprod/vm/flagstub/BUILD.bazel +++ b/pkg/roachprod/vm/flagstub/BUILD.bazel @@ -6,6 +6,7 @@ go_library( importpath = "github.com/cockroachdb/cockroach/pkg/roachprod/vm/flagstub", visibility = ["//visibility:public"], deps = [ + "//pkg/roachprod/logger", "//pkg/roachprod/vm", "@com_github_cockroachdb_errors//:errors", ], diff --git a/pkg/roachprod/vm/flagstub/flagstub.go b/pkg/roachprod/vm/flagstub/flagstub.go index 7a4f178fb624..ca88597f1ea6 100644 --- a/pkg/roachprod/vm/flagstub/flagstub.go +++ b/pkg/roachprod/vm/flagstub/flagstub.go @@ -13,6 +13,7 @@ package flagstub import ( "time" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/errors" ) @@ -42,7 +43,9 @@ func (p *provider) ConfigSSH(zones []string) error { } // Create implements vm.Provider and returns Unimplemented. -func (p *provider) Create(names []string, opts vm.CreateOpts, providerOpts vm.ProviderOpts) error { +func (p *provider) Create( + l *logger.Logger, names []string, opts vm.CreateOpts, providerOpts vm.ProviderOpts, +) error { return errors.Newf("%s", p.unimplemented) } @@ -67,7 +70,7 @@ func (p *provider) FindActiveAccount() (string, error) { } // List implements vm.Provider and returns an empty list. -func (p *provider) List() (vm.List, error) { +func (p *provider) List(l *logger.Logger) (vm.List, error) { return nil, nil } diff --git a/pkg/roachprod/vm/gce/BUILD.bazel b/pkg/roachprod/vm/gce/BUILD.bazel index 1803771de0d9..f56982dc0f2f 100644 --- a/pkg/roachprod/vm/gce/BUILD.bazel +++ b/pkg/roachprod/vm/gce/BUILD.bazel @@ -10,6 +10,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/roachprod/config", + "//pkg/roachprod/logger", "//pkg/roachprod/vm", "//pkg/roachprod/vm/flagstub", "//pkg/util/timeutil", diff --git a/pkg/roachprod/vm/gce/gcloud.go b/pkg/roachprod/vm/gce/gcloud.go index 9819fe4b7d46..5658588d2ac8 100644 --- a/pkg/roachprod/vm/gce/gcloud.go +++ b/pkg/roachprod/vm/gce/gcloud.go @@ -22,6 +22,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/roachprod/vm/flagstub" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -378,7 +379,7 @@ func (p *Provider) ConfigSSH(zones []string) error { // Create TODO(peter): document func (p *Provider) Create( - names []string, opts vm.CreateOpts, vmProviderOpts vm.ProviderOpts, + l *logger.Logger, names []string, opts vm.CreateOpts, vmProviderOpts vm.ProviderOpts, ) error { providerOpts := vmProviderOpts.(*ProviderOpts) project := p.GetProject() @@ -390,8 +391,8 @@ func (p *Provider) Create( } } if !gcJob { - fmt.Printf("WARNING: --lifetime functionality requires "+ - "`roachprod gc --gce-project=%s` cronjob\n", project) + l.Printf("WARNING: --lifetime functionality requires "+ + "`roachprod gc --gce-project=%s` cronjob", project) } zones, err := vm.ExpandZonesFlag(providerOpts.Zones) @@ -662,7 +663,7 @@ func (p *Provider) FindActiveAccount() (string, error) { } // List queries gcloud to produce a list of VM info objects. -func (p *Provider) List() (vm.List, error) { +func (p *Provider) List(l *logger.Logger) (vm.List, error) { var vms vm.List for _, prj := range p.GetProjects() { args := []string{"compute", "instances", "list", "--project", prj, "--format", "json"} diff --git a/pkg/roachprod/vm/gce/utils.go b/pkg/roachprod/vm/gce/utils.go index 78d9b42fed0f..90f10dece3a1 100644 --- a/pkg/roachprod/vm/gce/utils.go +++ b/pkg/roachprod/vm/gce/utils.go @@ -21,6 +21,7 @@ import ( "strings" "text/template" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/errors" ) @@ -249,7 +250,7 @@ func writeStartupScript( } // SyncDNS replaces the configured DNS zone with the supplied hosts. -func SyncDNS(vms vm.List) error { +func SyncDNS(l *logger.Logger, vms vm.List) error { if Subdomain == "" { return nil } @@ -261,7 +262,7 @@ func SyncDNS(vms vm.List) error { defer f.Close() defer func() { if err := os.Remove(f.Name()); err != nil { - fmt.Fprintf(os.Stderr, "removing %s failed: %v", f.Name(), err) + fmt.Fprintf(l.Stderr, "removing %s failed: %v", f.Name(), err) } }() @@ -269,7 +270,7 @@ func SyncDNS(vms vm.List) error { for _, vm := range vms { entry, err := vm.ZoneEntry() if err != nil { - fmt.Fprintf(os.Stderr, "WARN: skipping: %s\n", err) + fmt.Fprintf(l.Stderr, "WARN: skipping: %s\n", err) continue } zoneBuilder.WriteString(entry) diff --git a/pkg/roachprod/vm/local/BUILD.bazel b/pkg/roachprod/vm/local/BUILD.bazel index cc5b12b9d871..772be2af3a53 100644 --- a/pkg/roachprod/vm/local/BUILD.bazel +++ b/pkg/roachprod/vm/local/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//pkg/roachprod/cloud", "//pkg/roachprod/config", + "//pkg/roachprod/logger", "//pkg/roachprod/vm", "//pkg/util", "//pkg/util/timeutil", diff --git a/pkg/roachprod/vm/local/local.go b/pkg/roachprod/vm/local/local.go index 508edf865021..c7fc1124c5a3 100644 --- a/pkg/roachprod/vm/local/local.go +++ b/pkg/roachprod/vm/local/local.go @@ -19,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachprod/cloud" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -70,13 +71,13 @@ func AddCluster(cluster *cloud.Cluster) { // DeleteCluster destroys a local cluster. It assumes that the cockroach // processes are stopped. -func DeleteCluster(name string) error { +func DeleteCluster(l *logger.Logger, name string) error { p := vm.Providers[ProviderName].(*Provider) c := p.clusters[name] if c == nil { return fmt.Errorf("local cluster %s does not exist", name) } - fmt.Printf("Deleting local cluster %s\n", name) + l.Printf("Deleting local cluster %s\n", name) for i := range c.VMs { if err := os.RemoveAll(VMDir(c.Name, i+1)); err != nil { @@ -139,7 +140,7 @@ func (p *Provider) ConfigSSH(zones []string) error { // Create just creates fake host-info entries in the local filesystem func (p *Provider) Create( - names []string, opts vm.CreateOpts, unusedProviderOpts vm.ProviderOpts, + l *logger.Logger, names []string, opts vm.CreateOpts, unusedProviderOpts vm.ProviderOpts, ) error { now := timeutil.Now() c := &cloud.Cluster{ @@ -233,7 +234,7 @@ func (p *Provider) CreateProviderOpts() vm.ProviderOpts { } // List reports all the local cluster "VM" instances. -func (p *Provider) List() (vm.List, error) { +func (p *Provider) List(l *logger.Logger) (vm.List, error) { var result vm.List for _, clusterName := range p.clusters.Names() { c := p.clusters[clusterName] diff --git a/pkg/roachprod/vm/vm.go b/pkg/roachprod/vm/vm.go index 967e286633bb..97306d4a6461 100644 --- a/pkg/roachprod/vm/vm.go +++ b/pkg/roachprod/vm/vm.go @@ -19,6 +19,7 @@ import ( "unicode" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/errors" "github.com/spf13/pflag" "golang.org/x/sync/errgroup" @@ -238,13 +239,13 @@ type Provider interface { // ConfigSSH takes a list of zones and configures SSH for machines in those // zones for the given provider. ConfigSSH(zones []string) error - Create(names []string, opts CreateOpts, providerOpts ProviderOpts) error + Create(l *logger.Logger, names []string, opts CreateOpts, providerOpts ProviderOpts) error Reset(vms List) error Delete(vms List) error Extend(vms List, lifetime time.Duration) error // Return the account name associated with the provider FindActiveAccount() (string, error) - List() (List, error) + List(l *logger.Logger) (List, error) // The name of the Provider, which will also surface in the top-level Providers map. Name() string diff --git a/pkg/testutils/lint/lint_test.go b/pkg/testutils/lint/lint_test.go index 1a784daff64b..dbf2fbab3fca 100644 --- a/pkg/testutils/lint/lint_test.go +++ b/pkg/testutils/lint/lint_test.go @@ -1480,6 +1480,7 @@ func TestLint(t *testing.T) { stream.Uniq(), stream.Grep(`^`+settingsPkgPrefix+`: | `+grepBuf.String()), stream.GrepNot(`cockroach/pkg/cmd/`), + stream.GrepNot(`cockroach/pkg/roachprod/logger: log$`), stream.GrepNot(`cockroach/pkg/testutils/lint: log$`), stream.GrepNot(`cockroach/pkg/util/sysutil: syscall$`), stream.GrepNot(`cockroach/pkg/roachprod/install: syscall$`), // TODO: switch to sysutil @@ -2157,9 +2158,9 @@ func TestLint(t *testing.T) { // because addStructured takes its positional argument as []interface{}, // instead of ...interface{}. stream.GrepNot(`pkg/util/log/channels\.go:\d+:\d+: logfDepth\(\): format argument is not a constant expression`), - // roachtest is not collecting redactable logs so we don't care + // roachprod/logger is not collecting redactable logs so we don't care // about printf hygiene there as much. - stream.GrepNot(`pkg/cmd/roachtest/logger/log\.go:.*format argument is not a constant expression`), + stream.GrepNot(`pkg/roachprod/logger/log\.go:.*format argument is not a constant expression`), // We purposefully produce nil dereferences in this file to test crash conditions stream.GrepNot(`pkg/util/log/logcrash/crash_reporting_test\.go:.*nil dereference in type assertion`), // Spawning naked goroutines is ok when it's not as part of the main CRDB