Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Expose additional migration APIs #8153

Merged
merged 3 commits into from
Jul 30, 2021
Merged

Expose additional migration APIs #8153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bf9bd99
Expose additional migration APIs
gammazero May 25, 2021
1ca7b76
Add comment
gammazero Jun 24, 2021
b42c022
Review changes
gammazero Jul 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 1 addition & 138 deletions cmd/ipfs/migration.go → cmd/ipfs/add_migrations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,13 @@ package main

import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"strings"

config "github.com/ipfs/go-ipfs-config"
"github.com/ipfs/go-ipfs-files"
"github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/core/coreapi"
Expand All @@ -24,140 +20,7 @@ import (
"github.com/libp2p/go-libp2p-core/peer"
)

// readMigrationConfig reads the migration config out of the config, avoiding
// reading anything other than the migration section. That way, we're free to
// make arbitrary changes to all _other_ sections in migrations.
func readMigrationConfig(repoRoot string) (*config.Migration, error) {
var cfg struct {
Migration config.Migration
}

cfgPath, err := config.Filename(repoRoot)
if err != nil {
return nil, err
}

cfgFile, err := os.Open(cfgPath)
if err != nil {
return nil, err
}
defer cfgFile.Close()

err = json.NewDecoder(cfgFile).Decode(&cfg)
if err != nil {
return nil, err
}

switch cfg.Migration.Keep {
case "":
cfg.Migration.Keep = config.DefaultMigrationKeep
case "discard", "cache", "keep":
default:
return nil, errors.New("unknown config value, Migrations.Keep must be 'cache', 'pin', or 'discard'")
}

if len(cfg.Migration.DownloadSources) == 0 {
cfg.Migration.DownloadSources = config.DefaultMigrationDownloadSources
}

return &cfg.Migration, nil
}

func readIpfsConfig(repoRoot *string) (bootstrap []string, peers []peer.AddrInfo) {
if repoRoot == nil {
return
}

cfgPath, err := config.Filename(*repoRoot)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}

cfgFile, err := os.Open(cfgPath)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
defer cfgFile.Close()

// Attempt to read bootstrap addresses
var bootstrapCfg struct {
Bootstrap []string
}
err = json.NewDecoder(cfgFile).Decode(&bootstrapCfg)
if err != nil {
fmt.Fprintln(os.Stderr, "cannot read bootstrap peers from config")
} else {
bootstrap = bootstrapCfg.Bootstrap
}

if _, err = cfgFile.Seek(0, 0); err != nil {
fmt.Fprintln(os.Stderr, err)
}

// Attempt to read peers
var peeringCfg struct {
Peering config.Peering
}
err = json.NewDecoder(cfgFile).Decode(&peeringCfg)
if err != nil {
fmt.Fprintln(os.Stderr, "cannot read peering from config")
} else {
peers = peeringCfg.Peering.Peers
}

return
}

// getMigrationFetcher creates one or more fetchers according to
// config.Migration.DownloadSources. If an IpfsFetcher is required, then
// bootstrap and peer information in read from the config file in repoRoot,
// unless repoRoot is nil.
func getMigrationFetcher(cfg *config.Migration, repoRoot *string) (migrations.Fetcher, error) {
const httpUserAgent = "go-ipfs"

// Fetch migrations from current distribution, or location from environ
fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist)

var fetchers []migrations.Fetcher
for _, src := range cfg.DownloadSources {
src := strings.TrimSpace(src)
switch src {
case "IPFS", "ipfs":
bootstrap, peers := readIpfsConfig(repoRoot)
fetchers = append(fetchers, ipfsfetcher.NewIpfsFetcher(fetchDistPath, 0, bootstrap, peers))
case "HTTPS", "https", "HTTP", "http":
fetchers = append(fetchers, migrations.NewHttpFetcher(fetchDistPath, "", httpUserAgent, 0))
default:
u, err := url.Parse(src)
if err != nil {
return nil, fmt.Errorf("bad gateway address: %s", err)
}
switch u.Scheme {
case "":
u.Scheme = "https"
case "https", "http":
default:
return nil, errors.New("bad gateway address: url scheme must be http or https")
}
fetchers = append(fetchers, migrations.NewHttpFetcher(fetchDistPath, u.String(), httpUserAgent, 0))
case "":
// Ignore empty string
}
}
if len(fetchers) == 0 {
return nil, errors.New("no sources specified")
}

if len(fetchers) == 1 {
return fetchers[0], nil
}

// Wrap fetchers in a MultiFetcher to try them in order
return migrations.NewMultiFetcher(fetchers...), nil
}

// addMigrations adds any migration downloaded by the fetcher to the IPFS node
func addMigrations(ctx context.Context, node *core.IpfsNode, fetcher migrations.Fetcher, pin bool) error {
var fetchers []migrations.Fetcher
if mf, ok := fetcher.(*migrations.MultiFetcher); ok {
Expand Down
19 changes: 17 additions & 2 deletions cmd/ipfs/daemon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
nodeMount "github.com/ipfs/go-ipfs/fuse/node"
fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
"github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
"github.com/ipfs/go-ipfs/repo/fsrepo/migrations/ipfsfetcher"
sockets "github.com/libp2p/go-socket-activation"

cmds "github.com/ipfs/go-ipfs-cmds"
Expand Down Expand Up @@ -294,12 +295,26 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
return fmt.Errorf("fs-repo requires migration")
}

migrationCfg, err := readMigrationConfig(cctx.ConfigRoot)
// Read Migration section of IPFS config
migrationCfg, err := migrations.ReadMigrationConfig(cctx.ConfigRoot)
if err != nil {
return err
}

fetcher, err = getMigrationFetcher(migrationCfg, &cctx.ConfigRoot)
// Define function to create IPFS fetcher. Do not supply an
// already-constructed IPFS fetcher, because this may be expensive and
// not needed according to migration config. Instead, supply a function
// to construct the particular IPFS fetcher implementation used here,
// which is called only if an IPFS fetcher is needed.
newIpfsFetcher := func(distPath string) migrations.Fetcher {
return ipfsfetcher.NewIpfsFetcher(distPath, 0, &cctx.ConfigRoot)
}
gammazero marked this conversation as resolved.
Show resolved Hide resolved

// Fetch migrations from current distribution, or location from environ
fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist)

// Create fetchers according to migrationCfg.DownloadSources
fetcher, err = migrations.GetMigrationFetcher(migrationCfg.DownloadSources, fetchDistPath, newIpfsFetcher)
if err != nil {
return err
}
Expand Down
Loading