Framework that allows adding additional Quorum features as plugins (#923)

Author: trung

Makefile

@@ -195,4 +195,4 @@ endif
 ifeq (, $(shell which jq))
 	@echo "Please install jq from https://stedolan.github.io/jq/download"
 endif
-# QUORUM - END
+# QUORUM - END

cmd/geth/config.go

@@ -157,6 +157,12 @@ func enableWhisper(ctx *cli.Context) bool {
 func makeFullNode(ctx *cli.Context) *node.Node {
 	stack, cfg := makeConfigNode(ctx)
 
+	// this must be done first to make sure plugin manager is fully up.
+	// any fatal at this point is safe
+	if cfg.Node.Plugins != nil {
+		utils.RegisterPluginService(stack, &cfg.Node, ctx.Bool(utils.PluginSkipVerifyFlag.Name), ctx.Bool(utils.PluginLocalVerifyFlag.Name), ctx.String(utils.PluginPublicKeyFlag.Name))
+	}
+
 	ethChan := utils.RegisterEthService(stack, &cfg.Eth)
 
 	if cfg.Node.IsPermissionEnabled() {

cmd/geth/main.go

@@ -145,6 +145,10 @@ var (
 		utils.EmitCheckpointsFlag,
 		utils.IstanbulRequestTimeoutFlag,
 		utils.IstanbulBlockPeriodFlag,
+		utils.PluginSettingsFlag,
+		utils.PluginSkipVerifyFlag,
+		utils.PluginLocalVerifyFlag,
+		utils.PluginPublicKeyFlag,
 		// End-Quorum
 	}
 

cmd/geth/usage.go

@@ -141,6 +141,10 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "QUORUM",
 		Flags: []cli.Flag{
 			utils.EnableNodePermissionFlag,
+			utils.PluginSettingsFlag,
+			utils.PluginSkipVerifyFlag,
+			utils.PluginLocalVerifyFlag,
+			utils.PluginPublicKeyFlag,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -19,15 +19,19 @@ package utils
 
 import (
 	"crypto/ecdsa"
+	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/big"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/permission"
+	"github.com/ethereum/go-ethereum/plugin"
 
 	"time"
 
@@ -614,7 +618,23 @@ var (
 		Name:  "permissioned",
 		Usage: "If enabled, the node will allow only a defined list of nodes to connect",
 	}
-
+	// Plugins settings
+	PluginSettingsFlag = cli.StringFlag{
+		Name:  "plugins",
+		Usage: "The URI of configuration which describes plugins being used. E.g.: file:///opt/geth/plugins.json",
+	}
+	PluginLocalVerifyFlag = cli.BoolFlag{
+		Name:  "plugins.localverify",
+		Usage: "If enabled, verify plugin integrity from local file system. This requires plugin signature file and PGP public key file to be available",
+	}
+	PluginPublicKeyFlag = cli.StringFlag{
+		Name:  "plugins.publickey",
+		Usage: fmt.Sprintf("The URI of PGP public key for local plugin verification. E.g.: file:///opt/geth/pubkey.pgp.asc. This flag is only valid if --%s is set (default = file:///<pluginBaseDir>/%s)", PluginLocalVerifyFlag.Name, plugin.DefaultPublicKeyFile),
+	}
+	PluginSkipVerifyFlag = cli.BoolFlag{
+		Name:  "plugins.skipverify",
+		Usage: "If enabled, plugin integrity is NOT verified",
+	}
 	// Istanbul settings
 	IstanbulRequestTimeoutFlag = cli.Uint64Flag{
 		Name:  "istanbul.requesttimeout",
@@ -1053,6 +1073,51 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	if ctx.GlobalIsSet(NoUSBFlag.Name) {
 		cfg.NoUSB = ctx.GlobalBool(NoUSBFlag.Name)
 	}
+	if err := setPlugins(ctx, cfg); err != nil {
+		Fatalf(err.Error())
+	}
+}
+
+// Quorum
+//
+// Read plugin settings from --plugins flag. Overwrite settings defined in --config if any
+func setPlugins(ctx *cli.Context, cfg *node.Config) error {
+	if ctx.GlobalIsSet(PluginSettingsFlag.Name) {
+		// validate flag combination
+		if ctx.GlobalBool(PluginSkipVerifyFlag.Name) && ctx.GlobalBool(PluginLocalVerifyFlag.Name) {
+			return fmt.Errorf("only --%s or --%s must be set", PluginSkipVerifyFlag.Name, PluginLocalVerifyFlag.Name)
+		}
+		if !ctx.GlobalBool(PluginLocalVerifyFlag.Name) && ctx.GlobalIsSet(PluginPublicKeyFlag.Name) {
+			return fmt.Errorf("--%s is required for setting --%s", PluginLocalVerifyFlag.Name, PluginPublicKeyFlag.Name)
+		}
+		pluginSettingsURL, err := url.Parse(ctx.GlobalString(PluginSettingsFlag.Name))
+		if err != nil {
+			return fmt.Errorf("plugins: Invalid URL for --%s due to %s", PluginSettingsFlag.Name, err)
+		}
+		var pluginSettings plugin.Settings
+		r, err := urlReader(pluginSettingsURL)
+		if err != nil {
+			return fmt.Errorf("plugins: unable to create reader due to %s", err)
+		}
+		defer func() {
+			_ = r.Close()
+		}()
+		if err := json.NewDecoder(r).Decode(&pluginSettings); err != nil {
+			return fmt.Errorf("plugins: unable to parse settings due to %s", err)
+		}
+		pluginSettings.SetDefaults()
+		cfg.Plugins = &pluginSettings
+	}
+	return nil
+}
+
+func urlReader(u *url.URL) (io.ReadCloser, error) {
+	s := u.Scheme
+	switch s {
+	case "file":
+		return os.Open(filepath.Join(u.Host, u.Path))
+	}
+	return nil, fmt.Errorf("unsupported scheme %s", s)
 }
 
 func setGPO(ctx *cli.Context, cfg *gasprice.Config) {
@@ -1395,6 +1460,18 @@ func RegisterEthStatsService(stack *node.Node, url string) {
 
 // Quorum
 //
+// Register plugin manager as a service in geth
+func RegisterPluginService(stack *node.Node, cfg *node.Config, skipVerify bool, localVerify bool, publicKey string) {
+	if err := cfg.ResolvePluginBaseDir(); err != nil {
+		Fatalf("plugins: unable to resolve plugin base dir due to %s", err)
+	}
+	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		return plugin.NewPluginManager(cfg.UserIdent, cfg.Plugins, skipVerify, localVerify, publicKey)
+	}); err != nil {
+		Fatalf("plugins: Failed to register the Plugins service: %v", err)
+	}
+}
+
 // Configure smart-contract-based permissioning service
 func RegisterPermissionService(ctx *cli.Context, stack *node.Node) {
 	if err := stack.Register(func(sctx *node.ServiceContext) (node.Service, error) {

cmd/utils/flags_test.go

@@ -0,0 +1,79 @@
+package utils
+
+import (
+	"flag"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/urfave/cli.v1"
+)
+
+func TestSetPlugins_whenPluginsNotEnabled(t *testing.T) {
+	arbitraryNodeConfig := &node.Config{}
+	arbitraryCLIContext := cli.NewContext(nil, &flag.FlagSet{}, nil)
+
+	assert.NoError(t, setPlugins(arbitraryCLIContext, arbitraryNodeConfig))
+
+	assert.Nil(t, arbitraryNodeConfig.Plugins)
+}
+
+func TestSetPlugins_whenInvalidFlagsCombination(t *testing.T) {
+	arbitraryNodeConfig := &node.Config{}
+	fs := &flag.FlagSet{}
+	fs.String(PluginSettingsFlag.Name, "", "")
+	fs.Bool(PluginSkipVerifyFlag.Name, true, "")
+	fs.Bool(PluginLocalVerifyFlag.Name, true, "")
+	fs.String(PluginPublicKeyFlag.Name, "", "")
+	arbitraryCLIContext := cli.NewContext(nil, fs, nil)
+	assert.NoError(t, arbitraryCLIContext.GlobalSet(PluginSettingsFlag.Name, "arbitrary value"))
+
+	verifyErrorMessage(t, arbitraryCLIContext, arbitraryNodeConfig, "only --plugins.skipverify or --plugins.localverify must be set")
+
+	assert.NoError(t, arbitraryCLIContext.GlobalSet(PluginSkipVerifyFlag.Name, "false"))
+	assert.NoError(t, arbitraryCLIContext.GlobalSet(PluginLocalVerifyFlag.Name, "false"))
+	assert.NoError(t, arbitraryCLIContext.GlobalSet(PluginPublicKeyFlag.Name, "arbitry value"))
+
+	verifyErrorMessage(t, arbitraryCLIContext, arbitraryNodeConfig, "--plugins.localverify is required for setting --plugins.publickey")
+}
+
+func TestSetPlugins_whenInvalidPluginSettingsURL(t *testing.T) {
+	arbitraryNodeConfig := &node.Config{}
+	fs := &flag.FlagSet{}
+	fs.String(PluginSettingsFlag.Name, "", "")
+	arbitraryCLIContext := cli.NewContext(nil, fs, nil)
+	assert.NoError(t, arbitraryCLIContext.GlobalSet(PluginSettingsFlag.Name, "arbitrary value"))
+
+	verifyErrorMessage(t, arbitraryCLIContext, arbitraryNodeConfig, "plugins: unable to create reader due to unsupported scheme ")
+}
+
+func TestSetPlugins_whenTypical(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "q-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = os.RemoveAll(tmpDir)
+	}()
+	arbitraryJSONFile := path.Join(tmpDir, "arbitary.json")
+	if err := ioutil.WriteFile(arbitraryJSONFile, []byte("{}"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	arbitraryNodeConfig := &node.Config{}
+	fs := &flag.FlagSet{}
+	fs.String(PluginSettingsFlag.Name, "", "")
+	arbitraryCLIContext := cli.NewContext(nil, fs, nil)
+	assert.NoError(t, arbitraryCLIContext.GlobalSet(PluginSettingsFlag.Name, "file://"+arbitraryJSONFile))
+
+	assert.NoError(t, setPlugins(arbitraryCLIContext, arbitraryNodeConfig))
+
+	assert.NotNil(t, arbitraryNodeConfig.Plugins)
+}
+
+func verifyErrorMessage(t *testing.T, ctx *cli.Context, cfg *node.Config, expectedMsg string) {
+	err := setPlugins(ctx, cfg)
+	assert.EqualError(t, err, expectedMsg)
+}

docs/PluggableArchitecture/Internals.md

@@ -0,0 +1,142 @@
+title: Internals - Pluggable Architecture - Quorum
+
+## Background
+
+### Go Plugin
+`geth` is written in the Go programming language. [Go 1.8 introduced](https://golang.org/doc/go1.8#plugin) a new plugin architecture 
+which allows for the creation of plugins (via `plugin` build mode) and to use these plugins at runtime (via `plugin` package). 
+In order to utilize this architecture, there are strict requirements in developing plugins. 
+
+By using the network RPC interface, the plugin is independently built and distributed without having to rebuild `geth`. 
+Especially with gRPC interfaces, plugins can be written in different languages (see our [examples](../PluginDevelopment/#examples)).
+This makes it easy for you to build a prototype feature or even a proprietary plugin for your organization's internal use.
+
+We use HashiCorp's [`go-plugin`](https://github.com/hashicorp/go-plugin) library as it fits our asks 
+and it has been proven in many plugin-based production systems.
+
+### Why we decided to use plugins
+
+There are number of benefits:
+
+- Dynamically-linked binaries (which you get when using plugins) are much smaller than statically compiled binaries.
+- We value the ability to isolate failures. E.g.: Quorum client would continue mining/validating even if security plugin has crashed.
+- Easily enables support for open source plugins written in languages other than Go.
+
+## Design
+
+```plantuml
+skinparam componentStyle uml2
+skinparam shadowing false
+skinparam backgroundColor transparent
+skinparam rectangle {
+    roundCorner<<component>> 25
+}
+
+file "JSON File" as json
+file "TOML File" as toml
+note left of toml : Standard Ethereum Config
+note right of json : Quorum Plugin Settings
+
+node "geth" <<process>> {
+    rectangle "CLI Flags" as flags
+    frame "plugin.Settings" as settings {
+        storage "Plugin1\nDefinition" as pd1
+        storage "Plugin2\nDefinition" as pd2
+        storage "Plugin Central\nConnectivity" as pcc
+    }
+
+    json <-down- flags : "via\n""--plugins"""
+    toml <-down- flags : "via\n""--config"""
+    flags -down-> settings : populate
+
+    interface """node.Service""" as service
+    rectangle """plugin.PluginManager""" <<geth service>> as pm
+    note right of pm
+    registered and managed
+    as standard ""geth""
+    service life cycle
+    end note
+
+    pm -up- service
+    pm -up- settings
+
+    card "arbitrary" <<component>> as arbitrary
+    interface "internal1" as i1
+    interface "internal2" as i2
+    interface "internal3" as i3
+
+    package "Plugin Interface 1" {
+        rectangle "Plugin1" <<template>> as p1
+        rectangle "Gateway1" <<adapter>> as p1gw1
+        rectangle "Gateway2" <<adapter>> as p1gw2
+
+        interface "grpc service interface1A" as grpcI1A
+        interface "grpc service interface1B" as grpcI1B
+
+        rectangle "GRPC Stub Client1" <<grpc client>> as grpcC1
+    }
+    
+    package "Plugin Interface 2" {
+        rectangle "Plugin2" <<template>> as p2    
+        rectangle "Gateway" <<adapter>> as p2gw
+
+        interface "grpc service interface2" as grpcI2
+
+        rectangle "GRPC Stub Client2" <<grpc client>> as grpcC2
+    }
+
+    pm -- p1
+    pm -- p2
+
+    arbitrary --( i1
+    arbitrary --( i2
+    arbitrary --( i3
+
+    p1gw1 -- i1
+    p1gw2 -- i2
+    p2gw -- i3
+
+    p1 -- p1gw1
+    p1 -- p1gw2
+    p2 -- p2gw
+
+    grpcC1 --( grpcI1A
+    grpcC1 --( grpcI1B
+    grpcC2 --( grpcI2
+
+    p1gw1 --> grpcC1 : use
+    p1gw2 --> grpcC1 : use
+    p2gw --> grpcC2 : use
+}
+
+node "Plugin1" <<process>> {
+    rectangle "Implementation" <<grpc server>> as impl1
+}
+
+node "Plugin2" <<process>> {
+    rectangle "Implementation" <<grpc server>> as impl2
+}
+
+impl1 -up- grpcI1A
+impl1 -up- grpcI1B
+impl2 -up- grpcI2
+
+```
+
+### Discovery
+
+The Quorum client reads the plugin [settings](../Settings) file to determine which plugins are going to be loaded and searches for installed plugins
+(`<name>-<version>.zip` files) in the plugin `baseDir` (defaults to `<datadir>/plugins`). If the required plugin doesnt exist in the path, Quorum will attempt to use the configured `plugin central` to download the plugin.
+
+### PluginManager
+
+The `PluginManager` manages the plugins being used inside `geth`. It reads the [configuration](../Settings) and builds a registry of plugins.
+`PluginManager` implements the standard `Service` interface in `geth`, hence being embedded into the `geth` service life cycle, i.e.: expose service APIs, start and stop.
+The `PluginManager` service is registered as early as possible in the node lifecycle. This is to ensure the node fails fast if an issue is encountered when registering the `PluginManager`, so as not to impact other services.
+
+### Plugin Reloading
+
+The `PluginManager` exposes an API (`admin_reloadPlugin`) that allows reloading a plugin. This attempts to restart the current plugin process.   
+
+Any changes to the plugin config after initial node start will be applied when reloading the plugin.  
+This is demonstrated in the [HelloWorld plugin example](http://localhost:8000/PluggableArchitecture/Overview/#example-helloworld-plugin).