Release v0.35.0

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Kubo Changelogs
 
+- [v0.35](docs/changelogs/v0.35.md)
 - [v0.34](docs/changelogs/v0.34.md)
 - [v0.33](docs/changelogs/v0.33.md)
 - [v0.32](docs/changelogs/v0.32.md)

Dockerfile

@@ -80,8 +80,8 @@ RUN mkdir -p $IPFS_PATH \
   && chown ipfs:users $IPFS_PATH
 
 # Create mount points for `ipfs mount` command
-RUN mkdir /ipfs /ipns \
-  && chown ipfs:users /ipfs /ipns
+RUN mkdir /ipfs /ipns /mfs \
+  && chown ipfs:users /ipfs /ipns /mfs
 
 # Create the init scripts directory
 RUN mkdir /container-init.d \

cmd/ipfs/kubo/daemon.go

@@ -56,6 +56,7 @@ const (
 	initProfileOptionKwd       = "init-profile"
 	ipfsMountKwd               = "mount-ipfs"
 	ipnsMountKwd               = "mount-ipns"
+	mfsMountKwd                = "mount-mfs"
 	migrateKwd                 = "migrate"
 	mountKwd                   = "mount"
 	offlineKwd                 = "offline" // global option
@@ -173,6 +174,7 @@ Headers.
 		cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem using FUSE (experimental)"),
 		cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount). Defaults to config setting."),
 		cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount). Defaults to config setting."),
+		cmds.StringOption(mfsMountKwd, "Path to the mountpoint for MFS (if using --mount). Defaults to config setting."),
 		cmds.BoolOption(unrestrictedAPIAccessKwd, "Allow RPC API access to unlisted hashes"),
 		cmds.BoolOption(unencryptTransportKwd, "Disable transport encryption (for debugging protocols)"),
 		cmds.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection"),
@@ -458,6 +460,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 			cfg.Identity.PeerID,
 			cfg.Addresses,
 			cfg.Identity.PrivKey,
+			cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled),
 		)
 	default:
 		return fmt.Errorf("unrecognized routing option: %s", routingOption)
@@ -485,6 +488,14 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 		// This should never happen, but better safe than sorry
 		log.Fatal("Private network does not work with Routing.Type=auto. Update your config to Routing.Type=dht (or none, and do manual peering)")
 	}
+	if cfg.Provider.Strategy.WithDefault("") != "" && cfg.Reprovider.Strategy.IsDefault() {
+		log.Fatal("Invalid config. Remove unused Provider.Strategy and set Reprovider.Strategy instead. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy")
+	}
+	if cfg.Experimental.StrategicProviding {
+		log.Error("Experimental.StrategicProviding was removed. Remove it from your config and set Provider.Enabled=false to remove this message. Documentation: https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#strategic-providing")
+		cfg.Experimental.StrategicProviding = false
+		cfg.Provider.Enabled = config.False
+	}
 
 	printLibp2pPorts(node)
 
@@ -619,17 +630,19 @@ take effect.
 	}()
 
 	if !offline {
-		// Warn users who were victims of 'lowprofile' footgun (https://github.com/ipfs/kubo/pull/10524)
-		if cfg.Experimental.StrategicProviding {
+		// Warn users when provide systems are disabled
+		if !cfg.Provider.Enabled.WithDefault(config.DefaultProviderEnabled) {
 			fmt.Print(`
-⚠️ Reprovide system is disabled due to 'Experimental.StrategicProviding=true'
+
+⚠️ Provide and Reprovide systems are disabled due to 'Provide.Enabled=false'
 ⚠️ Local CIDs will not be announced to Amino DHT, making them impossible to retrieve without manual peering
-⚠️ If this is not intentional, call 'ipfs config profile apply announce-on'
+⚠️ If this is not intentional, call 'ipfs config profile apply announce-on' or set Provide.Enabled=true'
 
 `)
 		} else if cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval) == 0 {
 			fmt.Print(`
-⚠️ Reprovider system is disabled due to 'Reprovider.Interval=0'
+
+⚠️ Provide and Reprovide systems are disabled due to 'Reprovider.Interval=0'
 ⚠️ Local CIDs will not be announced to Amino DHT, making them impossible to retrieve without manual peering
 ⚠️ If this is not intentional, call 'ipfs config profile apply announce-on', or set 'Reprovider.Interval=22h'
 
@@ -1052,23 +1065,58 @@ func mountFuse(req *cmds.Request, cctx *oldcmds.Context) error {
 	if !found {
 		fsdir = cfg.Mounts.IPFS
 	}
+	if err := checkFusePath("Mounts.IPFS", fsdir); err != nil {
+		return err
+	}
 
 	nsdir, found := req.Options[ipnsMountKwd].(string)
 	if !found {
 		nsdir = cfg.Mounts.IPNS
 	}
+	if err := checkFusePath("Mounts.IPNS", nsdir); err != nil {
+		return err
+	}
+
+	mfsdir, found := req.Options[mfsMountKwd].(string)
+	if !found {
+		mfsdir = cfg.Mounts.MFS
+	}
+	if err := checkFusePath("Mounts.MFS", mfsdir); err != nil {
+		return err
+	}
 
 	node, err := cctx.ConstructNode()
 	if err != nil {
 		return fmt.Errorf("mountFuse: ConstructNode() failed: %s", err)
 	}
 
-	err = nodeMount.Mount(node, fsdir, nsdir)
+	err = nodeMount.Mount(node, fsdir, nsdir, mfsdir)
 	if err != nil {
 		return err
 	}
 	fmt.Printf("IPFS mounted at: %s\n", fsdir)
 	fmt.Printf("IPNS mounted at: %s\n", nsdir)
+	fmt.Printf("MFS mounted at: %s\n", mfsdir)
+	return nil
+}
+
+func checkFusePath(name, path string) error {
+	if path == "" {
+		return fmt.Errorf("%s path cannot be empty", name)
+	}
+
+	fileInfo, err := os.Stat(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("%s path (%q) does not exist: %w", name, path, err)
+		}
+		return fmt.Errorf("error while inspecting %s path (%q): %w", name, path, err)
+	}
+
+	if !fileInfo.IsDir() {
+		return fmt.Errorf("%s path (%q) is not a directory", name, path)
+	}
+
 	return nil
 }
 

config/bitswap.go

@@ -0,0 +1,15 @@
+package config
+
+// Bitswap holds Bitswap configuration options
+type Bitswap struct {
+	// Libp2pEnabled controls if the node initializes bitswap over libp2p (enabled by default)
+	// (This can be disabled if HTTPRetrieval.Enabled is set to true)
+	Libp2pEnabled Flag `json:",omitempty"`
+	// ServerEnabled controls if the node responds to WANTs (depends on Libp2pEnabled, enabled by default)
+	ServerEnabled Flag `json:",omitempty"`
+}
+
+const (
+	DefaultBitswapLibp2pEnabled = true
+	DefaultBitswapServerEnabled = true
+)

config/config.go

@@ -33,15 +33,18 @@ type Config struct {
 	DNS       DNS
 	Migration Migration
 
-	Provider     Provider
-	Reprovider   Reprovider
-	Experimental Experiments
-	Plugins      Plugins
-	Pinning      Pinning
-	Import       Import
-	Version      Version
+	Provider      Provider
+	Reprovider    Reprovider
+	HTTPRetrieval HTTPRetrieval
+	Experimental  Experiments
+	Plugins       Plugins
+	Pinning       Pinning
+	Import        Import
+	Version       Version
 
 	Internal Internal // experimental/unstable options
+
+	Bitswap Bitswap `json:",omitempty"`
 }
 
 const (

config/config_test.go

@@ -134,9 +134,9 @@ func TestCheckKey(t *testing.T) {
 		t.Fatal("Foo.Bar isn't a valid key in the config")
 	}
 
-	err = CheckKey("Provider.Strategy")
+	err = CheckKey("Reprovider.Strategy")
 	if err != nil {
-		t.Fatalf("%s: %s", err, "Provider.Strategy is a valid key in the config")
+		t.Fatalf("%s: %s", err, "Reprovider.Strategy is a valid key in the config")
 	}
 
 	err = CheckKey("Provider.Foo")

config/experiments.go

@@ -6,7 +6,7 @@ type Experiments struct {
 	ShardingEnabled               bool `json:",omitempty"` // deprecated by autosharding: https://github.com/ipfs/kubo/pull/8527
 	Libp2pStreamMounting          bool
 	P2pHttpProxy                  bool //nolint
-	StrategicProviding            bool
+	StrategicProviding            bool `json:",omitempty"` // removed, use Provider.Enabled instead
 	OptimisticProvide             bool
 	OptimisticProvideJobsPoolSize int
 	GatewayOverLibp2p             bool `json:",omitempty"`

config/http_retrieval.go

@@ -0,0 +1,19 @@
+package config
+
+// HTTPRetrieval is the configuration object for HTTP Retrieval settings.
+// Implicit defaults can be found in core/node/bitswap.go
+type HTTPRetrieval struct {
+	Enabled               Flag             `json:",omitempty"`
+	Allowlist             []string         `json:",omitempty"`
+	Denylist              []string         `json:",omitempty"`
+	NumWorkers            *OptionalInteger `json:",omitempty"`
+	MaxBlockSize          *OptionalString  `json:",omitempty"`
+	TLSInsecureSkipVerify Flag             `json:",omitempty"`
+}
+
+const (
+	DefaultHTTPRetrievalEnabled               = false // opt-in for now, until we figure out https://github.com/ipfs/specs/issues/496
+	DefaultHTTPRetrievalNumWorkers            = 16
+	DefaultHTTPRetrievalTLSInsecureSkipVerify = false  // only for testing with self-signed HTTPS certs
+	DefaultHTTPRetrievalMaxBlockSize          = "2MiB" // matching bitswap: https://specs.ipfs.tech/bitswap-protocol/#block-sizes
+)

config/import.go

@@ -1,11 +1,18 @@
 package config
 
+import (
+	"github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
+	"github.com/ipfs/boxo/ipld/unixfs/io"
+)
+
 const (
 	DefaultCidVersion      = 0
 	DefaultUnixFSRawLeaves = false
 	DefaultUnixFSChunker   = "size-262144"
 	DefaultHashFunction    = "sha2-256"
 
+	DefaultUnixFSHAMTDirectorySizeThreshold = "256KiB" // https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L26
+
 	// DefaultBatchMaxNodes controls the maximum number of nodes in a
 	// write-batch. The total size of the batch is limited by
 	// BatchMaxnodes and BatchMaxSize.
@@ -14,15 +21,26 @@ const (
 	// write-batch. The total size of the batch is limited by
 	// BatchMaxnodes and BatchMaxSize.
 	DefaultBatchMaxSize = 100 << 20 // 20MiB
+
+)
+
+var (
+	DefaultUnixFSFileMaxLinks           = int64(helpers.DefaultLinksPerBlock)
+	DefaultUnixFSDirectoryMaxLinks      = int64(0)
+	DefaultUnixFSHAMTDirectoryMaxFanout = int64(io.DefaultShardWidth)
 )
 
 // Import configures the default options for ingesting data. This affects commands
 // that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'.
 type Import struct {
-	CidVersion      OptionalInteger
-	UnixFSRawLeaves Flag
-	UnixFSChunker   OptionalString
-	HashFunction    OptionalString
-	BatchMaxNodes   OptionalInteger
-	BatchMaxSize    OptionalInteger
+	CidVersion                       OptionalInteger
+	UnixFSRawLeaves                  Flag
+	UnixFSChunker                    OptionalString
+	HashFunction                     OptionalString
+	UnixFSFileMaxLinks               OptionalInteger
+	UnixFSDirectoryMaxLinks          OptionalInteger
+	UnixFSHAMTDirectoryMaxFanout     OptionalInteger
+	UnixFSHAMTDirectorySizeThreshold OptionalString
+	BatchMaxNodes                    OptionalInteger
+	BatchMaxSize                     OptionalInteger
 }

config/init.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"time"
 
+	"github.com/cockroachdb/pebble/v2"
 	"github.com/ipfs/kubo/core/coreiface/options"
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -47,16 +48,11 @@ func InitWithIdentity(identity Identity) (*Config, error) {
 			},
 		},
 
-		Routing: Routing{
-			Type:    nil,
-			Methods: nil,
-			Routers: nil,
-		},
-
 		// setup the node mount points.
 		Mounts: Mounts{
 			IPFS: "/ipfs",
 			IPNS: "/ipns",
+			MFS:  "/mfs",
 		},
 
 		Ipns: Ipns{
@@ -139,17 +135,37 @@ func DefaultDatastoreConfig() Datastore {
 }
 
 func pebbleSpec() map[string]interface{} {
+	return map[string]interface{}{
+		"type":               "pebbleds",
+		"prefix":             "pebble.datastore",
+		"path":               "pebbleds",
+		"formatMajorVersion": int(pebble.FormatNewest),
+	}
+}
+
+func pebbleSpecMeasure() map[string]interface{} {
 	return map[string]interface{}{
 		"type":   "measure",
 		"prefix": "pebble.datastore",
 		"child": map[string]interface{}{
-			"type": "pebbleds",
-			"path": "pebbleds",
+			"formatMajorVersion": int(pebble.FormatNewest),
+			"type":               "pebbleds",
+			"path":               "pebbleds",
 		},
 	}
 }
 
 func badgerSpec() map[string]interface{} {
+	return map[string]interface{}{
+		"type":       "badgerds",
+		"prefix":     "badger.datastore",
+		"path":       "badgerds",
+		"syncWrites": false,
+		"truncate":   true,
+	}
+}
+
+func badgerSpecMeasure() map[string]interface{} {
 	return map[string]interface{}{
 		"type":   "measure",
 		"prefix": "badger.datastore",
@@ -163,6 +179,29 @@ func badgerSpec() map[string]interface{} {
 }
 
 func flatfsSpec() map[string]interface{} {
+	return map[string]interface{}{
+		"type": "mount",
+		"mounts": []interface{}{
+			map[string]interface{}{
+				"mountpoint": "/blocks",
+				"type":       "flatfs",
+				"prefix":     "flatfs.datastore",
+				"path":       "blocks",
+				"sync":       false,
+				"shardFunc":  "/repo/flatfs/shard/v1/next-to-last/2",
+			},
+			map[string]interface{}{
+				"mountpoint":  "/",
+				"type":        "levelds",
+				"prefix":      "leveldb.datastore",
+				"path":        "datastore",
+				"compression": "none",
+			},
+		},
+	}
+}
+
+func flatfsSpecMeasure() map[string]interface{} {
 	return map[string]interface{}{
 		"type": "mount",
 		"mounts": []interface{}{

config/internal.go

@@ -3,7 +3,7 @@ package config
 type Internal struct {
 	// All marked as omitempty since we are expecting to make changes to all subcomponents of Internal
 	Bitswap                     *InternalBitswap  `json:",omitempty"`
-	UnixFSShardingSizeThreshold *OptionalString   `json:",omitempty"`
+	UnixFSShardingSizeThreshold *OptionalString   `json:",omitempty"` // moved to Import.UnixFSHAMTDirectorySizeThreshold
 	Libp2pForceReachability     *OptionalString   `json:",omitempty"`
 	BackupBootstrapInterval     *OptionalDuration `json:",omitempty"`
 }
@@ -14,5 +14,6 @@ type InternalBitswap struct {
 	EngineTaskWorkerCount       OptionalInteger
 	MaxOutstandingBytesPerPeer  OptionalInteger
 	ProviderSearchDelay         OptionalDuration
+	ProviderSearchMaxResults    OptionalInteger
 	WantHaveReplaceSize         OptionalInteger
 }

config/mounts.go

@@ -4,5 +4,6 @@ package config
 type Mounts struct {
 	IPFS           string
 	IPNS           string
+	MFS            string
 	FuseAllowOther bool
 }