Merge pull request #4033 from ipfs/feat/git-plugin

 plugin: create plugin API and loader, add ipld-git plugin

Rules.mk

@@ -27,6 +27,10 @@ export IPFS_REUSEPORT=false
 dir := bin
 include $(dir)/Rules.mk
 
+# tests need access to rules from plugin
+dir := plugin
+include $(dir)/Rules.mk
+
 dir := test
 include $(dir)/Rules.mk
 
@@ -56,6 +60,7 @@ include $(dir)/Rules.mk
 dir := pin/internal/pb
 include $(dir)/Rules.mk
 
+
 # -------------------- #
 #   universal rules    #
 # -------------------- #
@@ -142,7 +147,7 @@ help:
 	@echo '  test_go_short'
 	@echo '  test_go_expensive'
 	@echo '  test_go_race'
-	@echo '  test_go_megacheck'	 - Run the `megacheck` vetting tool
+	@echo '  test_go_megacheck	 - Run the `megacheck` vetting tool'
 	@echo '  test_sharness_short'
 	@echo '  test_sharness_expensive'
 	@echo '  test_sharness_race'

cmd/ipfs/main.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime/pprof"
 	"strings"
 	"sync"
@@ -22,6 +23,7 @@ import (
 	cmdsHttp "github.com/ipfs/go-ipfs/commands/http"
 	core "github.com/ipfs/go-ipfs/core"
 	coreCmds "github.com/ipfs/go-ipfs/core/commands"
+	"github.com/ipfs/go-ipfs/plugin/loader"
 	repo "github.com/ipfs/go-ipfs/repo"
 	config "github.com/ipfs/go-ipfs/repo/config"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
@@ -339,6 +341,11 @@ func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd
 	} else {
 		log.Debug("executing command locally")
 
+		pluginpath := filepath.Join(req.InvocContext().ConfigRoot, "plugins")
+		if _, err := loader.LoadPlugins(pluginpath); err != nil {
+			return nil, err
+		}
+
 		err := req.SetRootContext(ctx)
 		if err != nil {
 			return nil, err

core/commands/dag/dag.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
+	coredag "github.com/ipfs/go-ipfs/core/coredag"
 	path "github.com/ipfs/go-ipfs/path"
 	pin "github.com/ipfs/go-ipfs/pin"
 
@@ -76,34 +77,25 @@ into an object of the specified format.
 			defer n.Blockstore.PinLock().Unlock()
 		}
 
-		var c *cid.Cid
-		switch ienc {
-		case "json":
-			nd, err := convertJsonToType(fi, format)
-			if err != nil {
-				res.SetError(err, cmds.ErrNormal)
-				return
-			}
+		nds, err := coredag.ParseInputs(ienc, format, fi)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
 
-			c, err = n.DAG.Add(nd)
-			if err != nil {
-				res.SetError(err, cmds.ErrNormal)
-				return
-			}
-		case "raw":
-			nd, err := convertRawToType(fi, format)
+		var c *cid.Cid
+		b := n.DAG.Batch()
+		for _, nd := range nds {
+			cid, err := b.Add(nd)
 			if err != nil {
 				res.SetError(err, cmds.ErrNormal)
 				return
 			}
 
-			c, err = n.DAG.Add(nd)
-			if err != nil {
-				res.SetError(err, cmds.ErrNormal)
-				return
-			}
-		default:
-			res.SetError(fmt.Errorf("unrecognized input encoding: %s", ienc), cmds.ErrNormal)
+			c = cid
+		}
+		if err := b.Commit(); err != nil {
+			res.SetError(err, cmds.ErrNormal)
 			return
 		}
 

core/coredag/dagtransl.go

@@ -0,0 +1,91 @@
+package coredag
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
+	ipldcbor "gx/ipfs/QmemYymP73eVdTUUMZEiSpiHeZQKNJdT5dP2iuHssZh1sR/go-ipld-cbor"
+)
+
+// DagParser is function used for parsing stream into Node
+type DagParser func(r io.Reader) ([]node.Node, error)
+
+// FormatParsers is used for mapping format descriptors to DagParsers
+type FormatParsers map[string]DagParser
+
+// InputEncParsers is used for mapping input encodings to FormatParsers
+type InputEncParsers map[string]FormatParsers
+
+// DefaultInputEncParsers is InputEncParser that is used everywhere
+var DefaultInputEncParsers = InputEncParsers{
+	"json": defaultJSONParsers,
+	"raw":  defaultRawParsers,
+}
+
+var defaultJSONParsers = FormatParsers{
+	"cbor":     cborJSONParser,
+	"dag-cbor": cborJSONParser,
+}
+
+var defaultRawParsers = FormatParsers{
+	"cbor":     cborRawParser,
+	"dag-cbor": cborRawParser,
+}
+
+// ParseInputs uses DefaultInputEncParsers to parse io.Reader described by
+// input encoding and format to an instance of ipld Node
+func ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
+	return DefaultInputEncParsers.ParseInputs(ienc, format, r)
+}
+
+// AddParser adds DagParser under give input encoding and format
+func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) {
+	m, ok := iep[ienv]
+	if !ok {
+		m = make(FormatParsers)
+		iep[ienv] = m
+	}
+
+	m[format] = f
+}
+
+// ParseInputs parses io.Reader described by input encoding and format to
+// an instance of ipld Node
+func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
+	pset, ok := iep[ienc]
+	if !ok {
+		return nil, fmt.Errorf("no input parser for %q", ienc)
+	}
+
+	parser, ok := pset[format]
+	if !ok {
+		return nil, fmt.Errorf("no parser for format %q using input type %q", format, ienc)
+	}
+
+	return parser(r)
+}
+
+func cborJSONParser(r io.Reader) ([]node.Node, error) {
+	nd, err := ipldcbor.FromJson(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return []node.Node{nd}, nil
+}
+
+func cborRawParser(r io.Reader) ([]node.Node, error) {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	nd, err := ipldcbor.Decode(data)
+	if err != nil {
+		return nil, err
+	}
+
+	return []node.Node{nd}, nil
+}

coverage/Rules.mk

@@ -1,6 +1,6 @@
 include mk/header.mk
 
-$(d)/coverage_deps:
+$(d)/coverage_deps: $$(DEPS_GO)
 	rm -rf $(@D)/unitcover && mkdir $(@D)/unitcover
 	rm -rf $(@D)/sharnesscover && mkdir $(@D)/sharnesscover
 ifneq ($(IPFS_SKIP_COVER_BINS),1)
@@ -41,6 +41,7 @@ endif
 
 export IPFS_COVER_DIR:= $(realpath $(d))/sharnesscover/
 
+$(d)/sharness_tests.coverprofile: export TEST_NO_PLUGIN=1
 $(d)/sharness_tests.coverprofile: $(d)/ipfs cmd/ipfs/ipfs-test-cover $(d)/coverage_deps test_sharness_short
 	(cd $(@D)/sharnesscover && find . -type f | gocovmerge -list -) > $@
 

mk/util.mk

@@ -1,4 +1,5 @@
 # util functions
+OS ?= $(shell sh -c 'uname -s 2>/dev/null || echo not')
 ifeq ($(OS),Windows_NT)
 	WINDOWS :=1
 	?exe :=.exe # windows compat

package.json

@@ -441,6 +441,12 @@
       "hash": "QmPjTrrSfE6TzLv6ya6VWhGcCgPrUAdcgrDcQyRDX2VyW1",
       "name": "go-libp2p-routing",
       "version": "2.2.17"
+    },
+    {
+      "author": "whyrusleeping",
+      "hash": "Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex",
+      "name": "go-ipld-git",
+      "version": "0.1.3"
     }
   ],
   "gxVersion": "0.10.0",

plugin/Rules.mk

@@ -0,0 +1,9 @@
+include mk/header.mk
+
+dir := $(d)/loader
+include $(dir)/Rules.mk
+
+dir := $(d)/plugins
+include $(dir)/Rules.mk
+
+include mk/footer.mk

plugin/ipld.go

@@ -0,0 +1,16 @@
+package plugin
+
+import (
+	"github.com/ipfs/go-ipfs/core/coredag"
+
+	node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
+)
+
+// PluginIPLD is an interface that can be implemented to add handlers for
+// for different IPLD formats
+type PluginIPLD interface {
+	Plugin
+
+	RegisterBlockDecoders(dec node.BlockDecoder) error
+	RegisterInputEncParsers(iec coredag.InputEncParsers) error
+}

plugin/loader/.gitignore

@@ -0,0 +1 @@
+preload.go

plugin/loader/Rules.mk

@@ -0,0 +1,10 @@
+include mk/header.mk
+
+$(d)/preload.go: d:=$(d)
+$(d)/preload.go: $(d)/preload_list
+	$(d)/preload.sh > $@
+	go fmt $@ >/dev/null
+
+DEPS_GO += $(d)/preload.go
+
+include mk/footer.mk

plugin/loader/initializer.go

@@ -0,0 +1,43 @@
+package loader
+
+import (
+	"github.com/ipfs/go-ipfs/core/coredag"
+	"github.com/ipfs/go-ipfs/plugin"
+
+	format "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
+)
+
+func initialize(plugins []plugin.Plugin) error {
+	for _, p := range plugins {
+		err := p.Init()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func run(plugins []plugin.Plugin) error {
+	for _, pl := range plugins {
+		err := runIPLDPlugin(pl)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func runIPLDPlugin(pl plugin.Plugin) error {
+	ipldpl, ok := pl.(plugin.PluginIPLD)
+	if !ok {
+		return nil
+	}
+
+	err := ipldpl.RegisterBlockDecoders(format.DefaultBlockDecoder)
+	if err != nil {
+		return err
+	}
+
+	return ipldpl.RegisterInputEncParsers(coredag.DefaultInputEncParsers)
+}

plugin/loader/load.go

@@ -0,0 +1,65 @@
+package loader
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ipfs/go-ipfs/plugin"
+
+	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
+)
+
+var log = logging.Logger("plugin/loader")
+
+var loadPluginsFunc = func(string) ([]plugin.Plugin, error) {
+	return nil, nil
+}
+
+// LoadPlugins loads and initializes plugins.
+func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) {
+	plMap := make(map[string]plugin.Plugin)
+	for _, v := range preloadPlugins {
+		plMap[v.Name()] = v
+	}
+
+	newPls, err := loadDynamicPlugins(pluginDir)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, pl := range newPls {
+		if ppl, ok := plMap[pl.Name()]; ok {
+			// plugin is already preloaded
+			return nil, fmt.Errorf(
+				"plugin: %s, is duplicated in version: %s, "+
+					"while trying to load dynamically: %s",
+				ppl.Name(), ppl.Version(), pl.Version())
+		}
+		plMap[pl.Name()] = pl
+	}
+
+	pls := make([]plugin.Plugin, 0, len(plMap))
+	for _, v := range plMap {
+		pls = append(pls, v)
+	}
+
+	err = initialize(pls)
+	if err != nil {
+		return nil, err
+	}
+
+	err = run(pls)
+	return nil, err
+}
+
+func loadDynamicPlugins(pluginDir string) ([]plugin.Plugin, error) {
+	_, err := os.Stat(pluginDir)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return loadPluginsFunc(pluginDir)
+}