Skip to content

Commit

Permalink
add remote pinning to ipfs command (#7661)
Browse files Browse the repository at this point in the history
Added support for remote pinning services

A pinning service is a service that accepts CIDs from a user in order to host the data associated with them.
The spec for these services is defined at https://github.com/ipfs/pinning-services-api-spec

Support is available via the `ipfs pin remote` CLI and the corresponding HTTP API

Co-authored-by: Petar Maymounkov <[email protected]>
Co-authored-by: Marcin Rataj <[email protected]>
Co-authored-by: Adin Schmahmann <[email protected]>
  • Loading branch information
3 people authored Dec 9, 2020
1 parent edde280 commit a8c7980
Show file tree
Hide file tree
Showing 10 changed files with 1,127 additions and 54 deletions.
32 changes: 30 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -118,14 +118,42 @@ jobs:
- store_artifacts:
path: /tmp/circleci-test-results
sharness:
executor: golang
machine:
image: ubuntu-2004:202010-01
working_directory: ~/ipfs/go-ipfs
environment:
<<: *default_environment
GO111MODULE: "on"
TEST_NO_DOCKER: 1
TEST_NO_FUSE: 1
GOPATH: /home/circleci/go
TEST_VERBOSE: 1
steps:
- run: sudo apt install socat
- checkout

- run:
mkdir rb-pinning-service-api &&
cd rb-pinning-service-api &&
git init &&
git remote add origin https://github.com/ipfs-shipyard/rb-pinning-service-api.git &&
git fetch --depth 1 origin 773c3adbb421c551d2d89288abac3e01e1f7c3a8 &&
git checkout FETCH_HEAD
- run:
cd rb-pinning-service-api &&
docker-compose pull &&
docker-compose up -d

- *make_out_dirs
- *restore_gomod

- run: make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1
- run:
name: Setup Environment Variables
# we need the docker host IP; all ports exported by child containers can be accessed there.
command: echo "export DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $BASH_ENV
- run:
echo DOCKER_HOST=$DOCKER_HOST &&
make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 DOCKER_HOST=$DOCKER_HOST

- run:
when: always
Expand Down
10 changes: 9 additions & 1 deletion core/commands/commands_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,11 +177,19 @@ func TestCommands(t *testing.T) {
"/p2p/stream/ls",
"/pin",
"/pin/add",
"/ping",
"/pin/ls",
"/pin/remote",
"/pin/remote/add",
"/pin/remote/ls",
"/pin/remote/rm",
"/pin/remote/service",
"/pin/remote/service/add",
"/pin/remote/service/ls",
"/pin/remote/service/rm",
"/pin/rm",
"/pin/update",
"/pin/verify",
"/ping",
"/pubsub",
"/pubsub/ls",
"/pubsub/peers",
Expand Down
80 changes: 75 additions & 5 deletions core/commands/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import (
"github.com/ipfs/go-ipfs/repo/fsrepo"

"github.com/elgris/jsondiff"
"github.com/ipfs/go-ipfs-cmds"
"github.com/ipfs/go-ipfs-config"
cmds "github.com/ipfs/go-ipfs-cmds"
config "github.com/ipfs/go-ipfs-config"
)

// ConfigUpdateOutput is config profile apply command's output
Expand All @@ -36,6 +36,8 @@ const (
configDryRunOptionName = "dry-run"
)

var tryRemoteServiceApiErr = errors.New("cannot show or change pinning services through this API (try: ipfs pin remote service --help)")

var ConfigCmd = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "Get and set ipfs config values.",
Expand Down Expand Up @@ -86,6 +88,12 @@ Set the value of the 'Datastore.Path' key:
default:
}

// Temporary fix until we move ApiKey secrets out of the config file
// (remote services are a map, so more advanced blocking is required)
if blocked := inBlockedScope(key, config.RemoteServicesSelector); blocked {
return tryRemoteServiceApiErr
}

cfgRoot, err := cmdenv.GetConfigRoot(env)
if err != nil {
return err
Expand Down Expand Up @@ -140,11 +148,29 @@ Set the value of the 'Datastore.Path' key:
Type: ConfigField{},
}

// Returns bool to indicate if tested key is in the blocked scope.
// (scope includes parent, direct, and child match)
func inBlockedScope(testKey string, blockedScope string) bool {
blockedScope = strings.ToLower(blockedScope)
roots := strings.Split(strings.ToLower(testKey), ".")
var scope []string
for _, name := range roots {
scope := append(scope, name)
impactedKey := strings.Join(scope, ".")
// blockedScope=foo.bar.BLOCKED should return true
// for parent and child impactedKeys: foo.bar and foo.bar.BLOCKED.subkey
if strings.HasPrefix(impactedKey, blockedScope) || strings.HasPrefix(blockedScope, impactedKey) {
return true
}
}
return false
}

var configShowCmd = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "Output config file contents.",
ShortDescription: `
NOTE: For security reasons, this command will omit your private key. If you would like to make a full backup of your config (private key included), you must copy the config file from your repo.
NOTE: For security reasons, this command will omit your private key and remote services. If you would like to make a full backup of your config (private key included), you must copy the config file from your repo.
`,
},
Type: map[string]interface{}{},
Expand Down Expand Up @@ -175,6 +201,11 @@ NOTE: For security reasons, this command will omit your private key. If you woul
return err
}

err = scrubOptionalValue(cfg, []string{config.PinningTag, config.RemoteServicesTag})
if err != nil {
return err
}

return cmds.EmitOnce(res, &cfg)
},
Encoders: cmds.EncoderMap{
Expand All @@ -190,7 +221,17 @@ NOTE: For security reasons, this command will omit your private key. If you woul
},
}

// Scrubs value and returns error if missing
func scrubValue(m map[string]interface{}, key []string) error {
return scrub(m, key, false)
}

// Scrubs value and returns no error if missing
func scrubOptionalValue(m map[string]interface{}, key []string) error {
return scrub(m, key, true)
}

func scrub(m map[string]interface{}, key []string, okIfMissing bool) error {
find := func(m map[string]interface{}, k string) (string, interface{}, bool) {
lckey := strings.ToLower(k)
for mkey, val := range m {
Expand All @@ -205,7 +246,7 @@ func scrubValue(m map[string]interface{}, key []string) error {
cur := m
for _, k := range key[:len(key)-1] {
foundk, val, ok := find(cur, k)
if !ok {
if !ok && !okIfMissing {
return errors.New("failed to find specified key")
}

Expand All @@ -223,7 +264,7 @@ func scrubValue(m map[string]interface{}, key []string) error {
}

todel, _, ok := find(cur, key[len(key)-1])
if !ok {
if !ok && !okIfMissing {
return fmt.Errorf("%s, not found", strings.Join(key, "."))
}

Expand Down Expand Up @@ -466,6 +507,9 @@ func replaceConfig(r repo.Repo, file io.Reader) error {
if err := json.NewDecoder(file).Decode(&cfg); err != nil {
return errors.New("failed to decode file as config")
}

// Handle Identity.PrivKey (secret)

if len(cfg.Identity.PrivKey) != 0 {
return errors.New("setting private key with API is not supported")
}
Expand All @@ -482,5 +526,31 @@ func replaceConfig(r repo.Repo, file io.Reader) error {

cfg.Identity.PrivKey = pkstr

// Handle Pinning.RemoteServices (ApiKey of each service is secret)
// Note: these settings are opt-in and may be missing

if len(cfg.Pinning.RemoteServices) != 0 {
return tryRemoteServiceApiErr
}

// detect if existing config has any remote services defined..
if remoteServicesTag, err := getConfig(r, config.RemoteServicesSelector); err == nil {
// seems that golang cannot type assert map[string]interface{} to map[string]config.RemotePinningService
// so we have to manually copy the data :-|
if val, ok := remoteServicesTag.Value.(map[string]interface{}); ok {
var services map[string]config.RemotePinningService
jsonString, err := json.Marshal(val)
if err != nil {
return fmt.Errorf("failed to replace config while preserving %s: %s", config.RemoteServicesSelector, err)
}
err = json.Unmarshal(jsonString, &services)
if err != nil {
return fmt.Errorf("failed to replace config while preserving %s: %s", config.RemoteServicesSelector, err)
}
// .. if so, apply them on top of the new config
cfg.Pinning.RemoteServices = services
}
}

return r.SetConfig(&cfg)
}
3 changes: 2 additions & 1 deletion core/commands/pin.go → core/commands/pin/pin.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package commands
package pin

import (
"context"
Expand Down Expand Up @@ -35,6 +35,7 @@ var PinCmd = &cmds.Command{
"ls": listPinCmd,
"verify": verifyPinCmd,
"update": updatePinCmd,
"remote": remotePinCmd,
},
}

Expand Down
Loading

0 comments on commit a8c7980

Please sign in to comment.