Merge pull request #1842 from hashicorp/backport/debug-plugin/presently-integral-jennet

Backport of Feature: plugin debugging into release/0.4.x

Author: izaaklauer

Diff for: .changelog/1716.txt

@@ -0,0 +1,4 @@
+```release-note:feature
+plugin: Allow debugging of plugins with tools like delve
+```
+

Diff for: go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/hashicorp/go-hclog v0.14.1
 	github.com/hashicorp/go-memdb v1.3.2
 	github.com/hashicorp/go-multierror v1.1.0
-	github.com/hashicorp/go-plugin v1.3.0
+	github.com/hashicorp/go-plugin v1.4.2
 	github.com/hashicorp/go-uuid v1.0.2
 	github.com/hashicorp/go-version v1.2.0
 	github.com/hashicorp/hcl/v2 v2.10.1-0.20210621220818-327f3ce2570e
@@ -59,7 +59,7 @@ require (
 	github.com/hashicorp/vault/api v1.0.5-0.20200519221902-385fac77e20f
 	github.com/hashicorp/vault/sdk v0.1.14-0.20201202172114-ee5ebeb30fef
 	github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9
-	github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210609145036-5c5b44751ee6
+	github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210625180209-eda7ae600c2d
 	github.com/imdario/mergo v0.3.11
 	github.com/improbable-eng/grpc-web v0.13.0
 	github.com/kevinburke/go-bindata v3.22.0+incompatible

Diff for: go.sum

@@ -822,8 +822,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY=
-github.com/hashicorp/go-plugin v1.3.0 h1:4d/wJojzvHV1I4i/rrjVaeuyxWrLzDE1mDCyDy8fXS8=
-github.com/hashicorp/go-plugin v1.3.0/go.mod h1:F9eH4LrE/ZsRdbwhfjs9k9HoDUwAHnYtXdgmf1AVNs0=
+github.com/hashicorp/go-plugin v1.4.2 h1:yFvG3ufXXpqiMiZx9HLcaK3XbIqQ1WJFR/F1a2CuVw0=
+github.com/hashicorp/go-plugin v1.4.2/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-retryablehttp v0.6.2/go.mod h1:gEx6HMUGxYYhJScX7W1Il64m6cc2C1mDaW3NQ9sY1FY=
@@ -882,8 +882,8 @@ github.com/hashicorp/vault/sdk v0.1.14-0.20201202172114-ee5ebeb30fef h1:YKouRHFf
 github.com/hashicorp/vault/sdk v0.1.14-0.20201202172114-ee5ebeb30fef/go.mod h1:cAGI4nVnEfAyMeqt9oB+Mase8DNn3qA/LDNHURiwssY=
 github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9 h1:i9hzlv2SpmaNcQ8ZLGn01fp2Gqyejj0juVs7rYDgecE=
 github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9/go.mod h1:ObgQSWSX9rsNofh16kctm6XxLW2QW1Ay6/9ris6T6DU=
-github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210609145036-5c5b44751ee6 h1:zhcfxphKiDYtZHmtgl4YC4rhNxIzv0aJmcz2fejJT2g=
-github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210609145036-5c5b44751ee6/go.mod h1:C6CXoZZFHsW8wb6WV6pIKZ5KFwd7XgK4GRD3hkt+lfg=
+github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210625180209-eda7ae600c2d h1:+ekQ8zfw9gM28KpRGXfEdrlR/JT8p4AiexfoBh+3bvE=
+github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210625180209-eda7ae600c2d/go.mod h1:l5rxDV4JfLCnMb/+0DxIjE40tHXPWbDyerz+f2Oul3I=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=

Diff for: internal/cli/plugin.go

@@ -1,25 +1,65 @@
 package cli
 
 import (
+	"context"
+	"fmt"
 	"github.com/hashicorp/waypoint-plugin-sdk"
+	"github.com/hashicorp/waypoint/internal/pkg/flag"
 	"github.com/hashicorp/waypoint/internal/plugin"
 )
 
 type PluginCommand struct {
 	*baseCommand
+	debugMode bool
 }
 
 func (c *PluginCommand) Run(args []string) int {
-	plugin, ok := plugin.Builtins[args[0]]
+	flags := c.Flags()
+
+	// Initialize. If we fail, we just exit since Init handles the UI.
+	if err := c.Init(
+		WithArgs(args),
+		WithFlags(flags),
+		WithClient(false),
+		WithNoAutoServer(),
+		WithNoConfig(),
+	); err != nil {
+		return 1
+	}
+	args = flags.Args()
+
+	pluginName := args[0]
+
+	plugin, ok := plugin.Builtins[pluginName]
 	if !ok {
-		panic("no such plugin: " + args[0])
+		panic("no such plugin: " + pluginName)
 	}
 
 	// Run the plugin
-	sdk.Main(plugin...)
+	if !c.debugMode {
+		sdk.Main(plugin...)
+	} else {
+		err := sdk.Debug(context.Background(), pluginName, plugin...)
+		if err != nil {
+			panic(fmt.Sprintf("Failed to launch plugin in debug mode: %s", err))
+		}
+	}
+
 	return 0
 }
 
+func (c *PluginCommand) Flags() *flag.Sets {
+	return c.flagSet(0, func(set *flag.Sets) {
+		f := set.NewSet("Command Options")
+		f.BoolVar(&flag.BoolVar{
+			Name:    "debug",
+			Default: false,
+			Target:  &c.debugMode,
+			Usage:   "set to true to run the plugin with support for debuggers like delve",
+		})
+	})
+}
+
 func (c *PluginCommand) Synopsis() string {
 	return "Execute a built-in plugin."
 }

Diff for: internal/client/server.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net"
 	"path/filepath"
@@ -93,7 +94,7 @@ func (c *Project) initLocalServer(ctx context.Context) (*grpc.ClientConn, error)
 		Timeout: 1 * time.Second,
 	})
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed opening boltdb path %s. Is another server already running against this db?: %w", path, err)
 	}
 	closers = append(closers, db)
 

Diff for: internal/plugin/factory.go

@@ -1,6 +1,7 @@
 package plugin
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
 	"runtime"
@@ -95,6 +96,65 @@ func BuiltinFactory(name string, typ component.Type) interface{} {
 	return Factory(cmd, typ)
 }
 
+// ReattachPluginFactory produces a provider factory that uses the passed
+// reattach information to connect to go-plugin processes that are already
+// running, and implements Instance against it.
+func ReattachPluginFactory(reattach *plugin.ReattachConfig, typ component.Type) interface{} {
+	return func(log hclog.Logger) (interface{}, error) {
+		config := pluginclient.ClientConfig(log)
+		config.Logger = log
+		config.Reattach = reattach
+
+		// Verify that we know about the requested plugin's protocol version
+		plugins, ok := config.VersionedPlugins[reattach.ProtocolVersion]
+		if !ok {
+			err := fmt.Errorf("no plugins found for requested protocol version number %d", reattach.ProtocolVersion)
+			log.Error(err.Error())
+			return Instance{}, err
+		}
+		config.Plugins = plugins
+
+		// Log that we're going to launch this
+		log.Info("Connecting to a reattach plugin", "type", typ, "Addr", reattach.Addr.String())
+
+		// Connect to the plugin
+		client := plugin.NewClient(config)
+		rpcClient, err := client.Client()
+		if err != nil {
+			log.Error("error creating reattach plugin client", "err", err)
+			client.Kill()
+			return nil, err
+		}
+
+		// Request the plugin. We don't request the mapper type because
+		// we handle that below.
+		var raw interface{}
+		if typ != component.MapperType {
+			raw, err = rpcClient.Dispense(strings.ToLower(typ.String()))
+			if err != nil {
+				log.Error("error requesting reattach plugin", "type", typ, "err", err)
+				client.Kill()
+				return nil, err
+			}
+		}
+
+		// Request the mappers
+		mappers, err := pluginclient.Mappers(client)
+		if err != nil {
+			log.Error("error requesting reattach plugin mappers", "err", err)
+			client.Kill()
+			return nil, err
+		}
+
+		log.Debug("successfully reattached to a plugin")
+		return &Instance{
+			Component: raw,
+			Mappers:   mappers,
+			Close:     func() { client.Kill() },
+		}, nil
+	}
+}
+
 // Instance is the result generated by the factory. This lets us pack
 // a bit more information into plugin-launched components.
 type Instance struct {

Diff for: internal/runner/operation.go

@@ -2,16 +2,22 @@ package runner
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
+	"os"
 	"path/filepath"
 	"reflect"
+	"strings"
 
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-multierror"
+	goplugin "github.com/hashicorp/go-plugin"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
+	sdk "github.com/hashicorp/waypoint-plugin-sdk"
 	"github.com/hashicorp/waypoint-plugin-sdk/component"
 	"github.com/hashicorp/waypoint-plugin-sdk/datadir"
 	"github.com/hashicorp/waypoint-plugin-sdk/terminal"
@@ -225,12 +231,33 @@ func (r *Runner) pluginFactories(
 	}
 	log.Debug("plugin search path", "path", pluginPaths)
 
+	// Look for any reattach plugins
+	var reattachPluginConfigs map[string]*goplugin.ReattachConfig
+	reattachPluginsStr := os.Getenv("WP_REATTACH_PLUGINS")
+	if reattachPluginsStr != "" {
+		var err error
+		reattachPluginConfigs, err = parseReattachPlugins(reattachPluginsStr)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Search for all of our plugins
 	var perr error
 	for _, pluginCfg := range plugins {
 		plog := log.With("plugin_name", pluginCfg.Name)
 		plog.Debug("searching for plugin")
 
+		if reattachConfig, ok := reattachPluginConfigs[pluginCfg.Name]; ok {
+			plog.Debug(fmt.Sprintf("plugin %s is declared as running for reattachment", pluginCfg.Name))
+			for _, t := range pluginCfg.Types() {
+				if err := result[t].Register(pluginCfg.Name, plugin.ReattachPluginFactory(reattachConfig, t)); err != nil {
+					return nil, err
+				}
+			}
+			continue
+		}
+
 		// Find our plugin.
 		cmd, err := plugin.Discover(pluginCfg, pluginPaths)
 		if err != nil {
@@ -267,6 +294,46 @@ func (r *Runner) pluginFactories(
 	return result, perr
 }
 
+// parse information on reattaching to plugins out of a
+// JSON-encoded environment variable.
+func parseReattachPlugins(in string) (map[string]*goplugin.ReattachConfig, error) {
+	reattachConfigs := map[string]*goplugin.ReattachConfig{}
+	if in != "" {
+		in = strings.TrimRight(in, "'")
+		in = strings.TrimLeft(in, "'")
+		var m map[string]sdk.ReattachConfig
+		err := json.Unmarshal([]byte(in), &m)
+		if err != nil {
+			return reattachConfigs, fmt.Errorf("Invalid format for WP_REATTACH_PROVIDERS: %w", err)
+		}
+		for p, c := range m {
+			var addr net.Addr
+			switch c.Addr.Network {
+			case "unix":
+				addr, err = net.ResolveUnixAddr("unix", c.Addr.String)
+				if err != nil {
+					return reattachConfigs, fmt.Errorf("Invalid unix socket path %q for %q: %w", c.Addr.String, p, err)
+				}
+			case "tcp":
+				addr, err = net.ResolveTCPAddr("tcp", c.Addr.String)
+				if err != nil {
+					return reattachConfigs, fmt.Errorf("Invalid TCP address %q for %q: %w", c.Addr.String, p, err)
+				}
+			default:
+				return reattachConfigs, fmt.Errorf("Unknown address type %q for %q", c.Addr.String, p)
+			}
+			reattachConfigs[p] = &goplugin.ReattachConfig{
+				Protocol:        goplugin.Protocol(c.Protocol),
+				ProtocolVersion: c.ProtocolVersion,
+				Pid:             c.Pid,
+				Test:            c.Test,
+				Addr:            addr,
+			}
+		}
+	}
+	return reattachConfigs, nil
+}
+
 // operationNoDataFunc is the function type for operations that are
 // executed without data downloaded.
 type operationNoDataFunc func(*Runner, context.Context, hclog.Logger, *pb.Job) (*pb.Job_Result, error)