Skip to content

Commit

Permalink
Merge pull request #4033 from ipfs/feat/git-plugin
Browse files Browse the repository at this point in the history
 plugin: create plugin API and loader, add ipld-git plugin
  • Loading branch information
whyrusleeping authored Jul 16, 2017
2 parents f16d942 + c0ee7d6 commit 71bb417
Show file tree
Hide file tree
Showing 26 changed files with 570 additions and 27 deletions.
7 changes: 6 additions & 1 deletion Rules.mk
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ export IPFS_REUSEPORT=false
dir := bin
include $(dir)/Rules.mk

# tests need access to rules from plugin
dir := plugin
include $(dir)/Rules.mk

dir := test
include $(dir)/Rules.mk

Expand Down Expand Up @@ -56,6 +60,7 @@ include $(dir)/Rules.mk
dir := pin/internal/pb
include $(dir)/Rules.mk


# -------------------- #
# universal rules #
# -------------------- #
Expand Down Expand Up @@ -142,7 +147,7 @@ help:
@echo ' test_go_short'
@echo ' test_go_expensive'
@echo ' test_go_race'
@echo ' test_go_megacheck' - Run the `megacheck` vetting tool
@echo ' test_go_megacheck - Run the `megacheck` vetting tool'
@echo ' test_sharness_short'
@echo ' test_sharness_expensive'
@echo ' test_sharness_race'
Expand Down
7 changes: 7 additions & 0 deletions cmd/ipfs/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"net/url"
"os"
"os/signal"
"path/filepath"
"runtime/pprof"
"strings"
"sync"
Expand All @@ -22,6 +23,7 @@ import (
cmdsHttp "github.com/ipfs/go-ipfs/commands/http"
core "github.com/ipfs/go-ipfs/core"
coreCmds "github.com/ipfs/go-ipfs/core/commands"
"github.com/ipfs/go-ipfs/plugin/loader"
repo "github.com/ipfs/go-ipfs/repo"
config "github.com/ipfs/go-ipfs/repo/config"
fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
Expand Down Expand Up @@ -339,6 +341,11 @@ func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd
} else {
log.Debug("executing command locally")

pluginpath := filepath.Join(req.InvocContext().ConfigRoot, "plugins")
if _, err := loader.LoadPlugins(pluginpath); err != nil {
return nil, err
}

err := req.SetRootContext(ctx)
if err != nil {
return nil, err
Expand Down
36 changes: 14 additions & 22 deletions core/commands/dag/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"strings"

cmds "github.com/ipfs/go-ipfs/commands"
coredag "github.com/ipfs/go-ipfs/core/coredag"
path "github.com/ipfs/go-ipfs/path"
pin "github.com/ipfs/go-ipfs/pin"

Expand Down Expand Up @@ -76,34 +77,25 @@ into an object of the specified format.
defer n.Blockstore.PinLock().Unlock()
}

var c *cid.Cid
switch ienc {
case "json":
nd, err := convertJsonToType(fi, format)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
nds, err := coredag.ParseInputs(ienc, format, fi)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}

c, err = n.DAG.Add(nd)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
case "raw":
nd, err := convertRawToType(fi, format)
var c *cid.Cid
b := n.DAG.Batch()
for _, nd := range nds {
cid, err := b.Add(nd)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}

c, err = n.DAG.Add(nd)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
default:
res.SetError(fmt.Errorf("unrecognized input encoding: %s", ienc), cmds.ErrNormal)
c = cid
}
if err := b.Commit(); err != nil {
res.SetError(err, cmds.ErrNormal)
return
}

Expand Down
91 changes: 91 additions & 0 deletions core/coredag/dagtransl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package coredag

import (
"fmt"
"io"
"io/ioutil"

node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
ipldcbor "gx/ipfs/QmemYymP73eVdTUUMZEiSpiHeZQKNJdT5dP2iuHssZh1sR/go-ipld-cbor"
)

// DagParser is function used for parsing stream into Node
type DagParser func(r io.Reader) ([]node.Node, error)

// FormatParsers is used for mapping format descriptors to DagParsers
type FormatParsers map[string]DagParser

// InputEncParsers is used for mapping input encodings to FormatParsers
type InputEncParsers map[string]FormatParsers

// DefaultInputEncParsers is InputEncParser that is used everywhere
var DefaultInputEncParsers = InputEncParsers{
"json": defaultJSONParsers,
"raw": defaultRawParsers,
}

var defaultJSONParsers = FormatParsers{
"cbor": cborJSONParser,
"dag-cbor": cborJSONParser,
}

var defaultRawParsers = FormatParsers{
"cbor": cborRawParser,
"dag-cbor": cborRawParser,
}

// ParseInputs uses DefaultInputEncParsers to parse io.Reader described by
// input encoding and format to an instance of ipld Node
func ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
return DefaultInputEncParsers.ParseInputs(ienc, format, r)
}

// AddParser adds DagParser under give input encoding and format
func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) {
m, ok := iep[ienv]
if !ok {
m = make(FormatParsers)
iep[ienv] = m
}

m[format] = f
}

// ParseInputs parses io.Reader described by input encoding and format to
// an instance of ipld Node
func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
pset, ok := iep[ienc]
if !ok {
return nil, fmt.Errorf("no input parser for %q", ienc)
}

parser, ok := pset[format]
if !ok {
return nil, fmt.Errorf("no parser for format %q using input type %q", format, ienc)
}

return parser(r)
}

func cborJSONParser(r io.Reader) ([]node.Node, error) {
nd, err := ipldcbor.FromJson(r)
if err != nil {
return nil, err
}

return []node.Node{nd}, nil
}

func cborRawParser(r io.Reader) ([]node.Node, error) {
data, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}

nd, err := ipldcbor.Decode(data)
if err != nil {
return nil, err
}

return []node.Node{nd}, nil
}
3 changes: 2 additions & 1 deletion coverage/Rules.mk
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
include mk/header.mk

$(d)/coverage_deps:
$(d)/coverage_deps: $$(DEPS_GO)
rm -rf $(@D)/unitcover && mkdir $(@D)/unitcover
rm -rf $(@D)/sharnesscover && mkdir $(@D)/sharnesscover
ifneq ($(IPFS_SKIP_COVER_BINS),1)
Expand Down Expand Up @@ -41,6 +41,7 @@ endif

export IPFS_COVER_DIR:= $(realpath $(d))/sharnesscover/

$(d)/sharness_tests.coverprofile: export TEST_NO_PLUGIN=1
$(d)/sharness_tests.coverprofile: $(d)/ipfs cmd/ipfs/ipfs-test-cover $(d)/coverage_deps test_sharness_short
(cd $(@D)/sharnesscover && find . -type f | gocovmerge -list -) > $@

Expand Down
1 change: 1 addition & 0 deletions mk/util.mk
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# util functions
OS ?= $(shell sh -c 'uname -s 2>/dev/null || echo not')
ifeq ($(OS),Windows_NT)
WINDOWS :=1
?exe :=.exe # windows compat
Expand Down
6 changes: 6 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,12 @@
"hash": "QmPjTrrSfE6TzLv6ya6VWhGcCgPrUAdcgrDcQyRDX2VyW1",
"name": "go-libp2p-routing",
"version": "2.2.17"
},
{
"author": "whyrusleeping",
"hash": "Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex",
"name": "go-ipld-git",
"version": "0.1.3"
}
],
"gxVersion": "0.10.0",
Expand Down
9 changes: 9 additions & 0 deletions plugin/Rules.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
include mk/header.mk

dir := $(d)/loader
include $(dir)/Rules.mk

dir := $(d)/plugins
include $(dir)/Rules.mk

include mk/footer.mk
16 changes: 16 additions & 0 deletions plugin/ipld.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package plugin

import (
"github.com/ipfs/go-ipfs/core/coredag"

node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
)

// PluginIPLD is an interface that can be implemented to add handlers for
// for different IPLD formats
type PluginIPLD interface {
Plugin

RegisterBlockDecoders(dec node.BlockDecoder) error
RegisterInputEncParsers(iec coredag.InputEncParsers) error
}
1 change: 1 addition & 0 deletions plugin/loader/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
preload.go
10 changes: 10 additions & 0 deletions plugin/loader/Rules.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
include mk/header.mk

$(d)/preload.go: d:=$(d)
$(d)/preload.go: $(d)/preload_list
$(d)/preload.sh > $@
go fmt $@ >/dev/null

DEPS_GO += $(d)/preload.go

include mk/footer.mk
43 changes: 43 additions & 0 deletions plugin/loader/initializer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package loader

import (
"github.com/ipfs/go-ipfs/core/coredag"
"github.com/ipfs/go-ipfs/plugin"

format "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
)

func initialize(plugins []plugin.Plugin) error {
for _, p := range plugins {
err := p.Init()
if err != nil {
return err
}
}

return nil
}

func run(plugins []plugin.Plugin) error {
for _, pl := range plugins {
err := runIPLDPlugin(pl)
if err != nil {
return err
}
}
return nil
}

func runIPLDPlugin(pl plugin.Plugin) error {
ipldpl, ok := pl.(plugin.PluginIPLD)
if !ok {
return nil
}

err := ipldpl.RegisterBlockDecoders(format.DefaultBlockDecoder)
if err != nil {
return err
}

return ipldpl.RegisterInputEncParsers(coredag.DefaultInputEncParsers)
}
65 changes: 65 additions & 0 deletions plugin/loader/load.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package loader

import (
"fmt"
"os"

"github.com/ipfs/go-ipfs/plugin"

logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
)

var log = logging.Logger("plugin/loader")

var loadPluginsFunc = func(string) ([]plugin.Plugin, error) {
return nil, nil
}

// LoadPlugins loads and initializes plugins.
func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) {
plMap := make(map[string]plugin.Plugin)
for _, v := range preloadPlugins {
plMap[v.Name()] = v
}

newPls, err := loadDynamicPlugins(pluginDir)
if err != nil {
return nil, err
}

for _, pl := range newPls {
if ppl, ok := plMap[pl.Name()]; ok {
// plugin is already preloaded
return nil, fmt.Errorf(
"plugin: %s, is duplicated in version: %s, "+
"while trying to load dynamically: %s",
ppl.Name(), ppl.Version(), pl.Version())
}
plMap[pl.Name()] = pl
}

pls := make([]plugin.Plugin, 0, len(plMap))
for _, v := range plMap {
pls = append(pls, v)
}

err = initialize(pls)
if err != nil {
return nil, err
}

err = run(pls)
return nil, err
}

func loadDynamicPlugins(pluginDir string) ([]plugin.Plugin, error) {
_, err := os.Stat(pluginDir)
if os.IsNotExist(err) {
return nil, nil
}
if err != nil {
return nil, err
}

return loadPluginsFunc(pluginDir)
}
Loading

0 comments on commit 71bb417

Please sign in to comment.