From 05d0bb329006e3b51424a8c2f97dc9c23165cf63 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Wed, 9 Apr 2025 14:44:42 +0200 Subject: [PATCH 01/14] adjust ipfs stats provide --- core/commands/stat_provide.go | 8 ++++---- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/commands/stat_provide.go b/core/commands/stat_provide.go index 8a3dcff305a..1487ae36dde 100644 --- a/core/commands/stat_provide.go +++ b/core/commands/stat_provide.go @@ -57,13 +57,13 @@ This interface is not stable and may change from release to release. wtr := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) defer wtr.Flush() - fmt.Fprintf(wtr, "TotalProvides:\t%s\n", humanNumber(s.TotalProvides)) - fmt.Fprintf(wtr, "AvgProvideDuration:\t%s\n", humanDuration(s.AvgProvideDuration)) + fmt.Fprintf(wtr, "TotalReprovides:\t%s\n", humanNumber(s.TotalReprovides)) + fmt.Fprintf(wtr, "AvgReprovideDuration:\t%s\n", humanDuration(s.AvgReprovideDuration)) fmt.Fprintf(wtr, "LastReprovideDuration:\t%s\n", humanDuration(s.LastReprovideDuration)) if !s.LastRun.IsZero() { - fmt.Fprintf(wtr, "LastRun:\t%s\n", humanTime(s.LastRun)) + fmt.Fprintf(wtr, "LastReprovide:\t%s\n", humanTime(s.LastRun)) if s.fullRT { - fmt.Fprintf(wtr, "NextRun:\t%s\n", humanTime(s.LastRun.Add(s.ReprovideInterval))) + fmt.Fprintf(wtr, "NextReprovide:\t%s\n", humanTime(s.LastRun.Add(s.ReprovideInterval))) } } return nil diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 38b3e64e9a1..9add2d136b0 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.24 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.29.1 + github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.41.1 github.com/multiformats/go-multiaddr v0.15.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index f8fc2a88567..bc2b8ebc497 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc= -github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU= +github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 h1:4NEUKdqzASso0L4rN/Cx/b4st4anWrd/jZ9uHOe+/3Q= +github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/go.mod b/go.mod index 1abbe78123d..261a3d3b0b6 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.29.1 + github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 91d34dbbdb0..0995a8cbf56 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc= -github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU= +github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 h1:4NEUKdqzASso0L4rN/Cx/b4st4anWrd/jZ9uHOe+/3Q= +github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 3f6a1e39bcd..3ebc9b6e94c 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -116,7 +116,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.29.1 // indirect + github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cid v0.5.0 // indirect github.com/ipfs/go-datastore v0.8.2 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 757ab2b7c52..bb64c1ee9f6 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc= -github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU= +github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 h1:4NEUKdqzASso0L4rN/Cx/b4st4anWrd/jZ9uHOe+/3Q= +github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= From 4e8d223a3c165c95ad1344d7b26a2aac69910c0c Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Wed, 16 Apr 2025 10:20:42 +0200 Subject: [PATCH 02/14] update boxo dep --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 9add2d136b0..6a2b1e18197 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.24 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 + github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.41.1 github.com/multiformats/go-multiaddr v0.15.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index bc2b8ebc497..67c167d69ca 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 h1:4NEUKdqzASso0L4rN/Cx/b4st4anWrd/jZ9uHOe+/3Q= -github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 h1:7IZ+dJCQSHRuPKkogansDPNRKS3rn18a0uaHJl2mRpw= +github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/go.mod b/go.mod index 261a3d3b0b6..b97e67d7234 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 + github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 0995a8cbf56..4b23827f788 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 h1:4NEUKdqzASso0L4rN/Cx/b4st4anWrd/jZ9uHOe+/3Q= -github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 h1:7IZ+dJCQSHRuPKkogansDPNRKS3rn18a0uaHJl2mRpw= +github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 3ebc9b6e94c..3dc7940efa5 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -116,7 +116,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 // indirect + github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cid v0.5.0 // indirect github.com/ipfs/go-datastore v0.8.2 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index bb64c1ee9f6..be2146606e4 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25 h1:4NEUKdqzASso0L4rN/Cx/b4st4anWrd/jZ9uHOe+/3Q= -github.com/ipfs/boxo v0.29.2-0.20250409143222-7a8f29cb4c25/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 h1:7IZ+dJCQSHRuPKkogansDPNRKS3rn18a0uaHJl2mRpw= +github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= From 62d51d6dd665d533e797d7472ad096cb703e1874 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Wed, 16 Apr 2025 10:42:51 +0200 Subject: [PATCH 03/14] update boxo dep --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 3 ++- test/dependencies/go.sum | 6 ++++-- 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 6a2b1e18197..8c87d9e8774 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.24 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 + github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.41.1 github.com/multiformats/go-multiaddr v0.15.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 67c167d69ca..39c3acde825 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 h1:7IZ+dJCQSHRuPKkogansDPNRKS3rn18a0uaHJl2mRpw= -github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c h1:ezJo/ZSX1elM5SKyNI4YUtGJBdECY78xv/ZmzS9xWaE= +github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/go.mod b/go.mod index b97e67d7234..3002a068d6a 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 + github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 4b23827f788..3a25b5b23f2 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 h1:7IZ+dJCQSHRuPKkogansDPNRKS3rn18a0uaHJl2mRpw= -github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c h1:ezJo/ZSX1elM5SKyNI4YUtGJBdECY78xv/ZmzS9xWaE= +github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index be63a2ea1ad..398840cd9e4 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -118,7 +118,8 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 // indirect + github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c // indirect + github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cid v0.5.0 // indirect github.com/ipfs/go-datastore v0.8.2 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 824cd534809..f99da635005 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -298,8 +298,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3 h1:7IZ+dJCQSHRuPKkogansDPNRKS3rn18a0uaHJl2mRpw= -github.com/ipfs/boxo v0.29.2-0.20250416081723-47720cbbd8d3/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c h1:ezJo/ZSX1elM5SKyNI4YUtGJBdECY78xv/ZmzS9xWaE= +github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= +github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= From b33c4fa970692ccabd6b566f1bb20df2759e200a Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Wed, 16 Apr 2025 11:26:09 +0200 Subject: [PATCH 04/14] provider worker count config --- config/init.go | 3 +++ config/provider.go | 7 ++++++- core/node/groups.go | 1 + core/node/provider.go | 8 ++++---- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/config/init.go b/config/init.go index a0351bd8b18..2157b469e68 100644 --- a/config/init.go +++ b/config/init.go @@ -69,6 +69,9 @@ func InitWithIdentity(identity Identity) (*Config, error) { NoFetch: false, HTTPHeaders: map[string][]string{}, }, + Provider: Provider{ + WorkerCount: 64, + }, Reprovider: Reprovider{ Interval: nil, Strategy: nil, diff --git a/config/provider.go b/config/provider.go index f2b5afe05b4..191d146594d 100644 --- a/config/provider.go +++ b/config/provider.go @@ -1,5 +1,10 @@ package config +const ( + DefaultProviderWorkerCount = 64 +) + type Provider struct { - Strategy string // Which keys to announce + Strategy string // Which keys to announce + WorkerCount uint // Number of concurrent provides allowed, 0 means unlimited } diff --git a/core/node/groups.go b/core/node/groups.go index 4a471f17038..a9591496606 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -359,6 +359,7 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy), cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval), cfg.Routing.AcceleratedDHTClient.WithDefault(config.DefaultAcceleratedDHTClient), + cfg.Provider.WorkerCount, ), ) } diff --git a/core/node/provider.go b/core/node/provider.go index 4638aad4dc5..8216b0b1495 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -21,12 +21,13 @@ import ( // and in 'ipfs stats provide' report. const sampledBatchSize = 1000 -func ProviderSys(reprovideInterval time.Duration, acceleratedDHTClient bool) fx.Option { +func ProviderSys(reprovideInterval time.Duration, acceleratedDHTClient bool, provideWorkerCount uint) fx.Option { return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo, bs blockstore.Blockstore) (provider.System, error) { opts := []provider.Option{ provider.Online(cr), provider.ReproviderInterval(reprovideInterval), provider.KeyProvider(keyProvider), + provider.ProvideWorkerCount(provideWorkerCount), } if !acceleratedDHTClient && reprovideInterval > 0 { // The estimation kinda suck if you are running with accelerated DHT client, @@ -131,7 +132,7 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtcli // ONLINE/OFFLINE // OnlineProviders groups units managing provider routing records online -func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval time.Duration, acceleratedDHTClient bool) fx.Option { +func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval time.Duration, acceleratedDHTClient bool, provideWorkerCount uint) fx.Option { if useStrategicProviding { return OfflineProviders() } @@ -146,7 +147,7 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro return fx.Options( keyProvider, - ProviderSys(reprovideInterval, acceleratedDHTClient), + ProviderSys(reprovideInterval, acceleratedDHTClient, provideWorkerCount), ) } @@ -169,7 +170,6 @@ func mfsProvider(mfsRoot *mfs.Root, fetcher fetcher.Factory) provider.KeyChanFun kcf := provider.NewDAGProvider(rootNode.Cid(), fetcher) return kcf(ctx) } - } func mfsRootProvider(mfsRoot *mfs.Root) provider.KeyChanFunc { From b0ba27c4e64fbaf8d755853b1a009fd385273130 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Wed, 16 Apr 2025 12:16:21 +0200 Subject: [PATCH 05/14] adjust config logic --- config/init.go | 3 --- config/provider.go | 4 ++-- core/node/groups.go | 2 +- core/node/provider.go | 4 ++-- docs/config.md | 10 ++++++++++ docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 11 files changed, 24 insertions(+), 17 deletions(-) diff --git a/config/init.go b/config/init.go index 2157b469e68..a0351bd8b18 100644 --- a/config/init.go +++ b/config/init.go @@ -69,9 +69,6 @@ func InitWithIdentity(identity Identity) (*Config, error) { NoFetch: false, HTTPHeaders: map[string][]string{}, }, - Provider: Provider{ - WorkerCount: 64, - }, Reprovider: Reprovider{ Interval: nil, Strategy: nil, diff --git a/config/provider.go b/config/provider.go index 191d146594d..b62be9499db 100644 --- a/config/provider.go +++ b/config/provider.go @@ -5,6 +5,6 @@ const ( ) type Provider struct { - Strategy string // Which keys to announce - WorkerCount uint // Number of concurrent provides allowed, 0 means unlimited + Strategy string // Which keys to announce + WorkerCount OptionalInteger // Number of concurrent provides allowed, 0 means unlimited } diff --git a/core/node/groups.go b/core/node/groups.go index a9591496606..0e28444be95 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -359,7 +359,7 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy), cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval), cfg.Routing.AcceleratedDHTClient.WithDefault(config.DefaultAcceleratedDHTClient), - cfg.Provider.WorkerCount, + int(cfg.Provider.WorkerCount.WithDefault(config.DefaultProviderWorkerCount)), ), ) } diff --git a/core/node/provider.go b/core/node/provider.go index 8216b0b1495..d0081eb0aec 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -21,7 +21,7 @@ import ( // and in 'ipfs stats provide' report. const sampledBatchSize = 1000 -func ProviderSys(reprovideInterval time.Duration, acceleratedDHTClient bool, provideWorkerCount uint) fx.Option { +func ProviderSys(reprovideInterval time.Duration, acceleratedDHTClient bool, provideWorkerCount int) fx.Option { return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo, bs blockstore.Blockstore) (provider.System, error) { opts := []provider.Option{ provider.Online(cr), @@ -132,7 +132,7 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtcli // ONLINE/OFFLINE // OnlineProviders groups units managing provider routing records online -func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval time.Duration, acceleratedDHTClient bool, provideWorkerCount uint) fx.Option { +func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval time.Duration, acceleratedDHTClient bool, provideWorkerCount int) fx.Option { if useStrategicProviding { return OfflineProviders() } diff --git a/docs/config.md b/docs/config.md index 1ec5677db28..66b72c8c6b5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -105,6 +105,8 @@ config file at runtime. - [`Pinning.RemoteServices: Policies.MFS.Enabled`](#pinningremoteservices-policiesmfsenabled) - [`Pinning.RemoteServices: Policies.MFS.PinName`](#pinningremoteservices-policiesmfspinname) - [`Pinning.RemoteServices: Policies.MFS.RepinInterval`](#pinningremoteservices-policiesmfsrepininterval) + - [`Provider`](#provider) + - [`Provider.WorkerCount`](#providerworkercount) - [`Pubsub`](#pubsub) - [`Pubsub.Enabled`](#pubsubenabled) - [`Pubsub.Router`](#pubsubrouter) @@ -1404,6 +1406,14 @@ Default: `"5m"` Type: `duration` +## `Provider` + +### `Provider.WorkerCount` + +Default: `64` + +Type: `integer` (non-negative, 0 means unlimited number of workers) + ## `Pubsub` **DEPRECATED**: See [#9717](https://github.com/ipfs/kubo/issues/9717) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 8c87d9e8774..2ddc3c12190 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.24 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c + github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.41.1 github.com/multiformats/go-multiaddr v0.15.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 39c3acde825..18716e28909 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c h1:ezJo/ZSX1elM5SKyNI4YUtGJBdECY78xv/ZmzS9xWaE= -github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 h1:ByhD/CZId7jjkMLKBi0euA8vEZD6Bma9rysZqSXQuxM= +github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/go.mod b/go.mod index 3002a068d6a..10db23aefee 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c + github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 3a25b5b23f2..cbeb7cef942 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c h1:ezJo/ZSX1elM5SKyNI4YUtGJBdECY78xv/ZmzS9xWaE= -github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 h1:ByhD/CZId7jjkMLKBi0euA8vEZD6Bma9rysZqSXQuxM= +github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 398840cd9e4..f5379818895 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -118,7 +118,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c // indirect + github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cid v0.5.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index f99da635005..c30d121dec4 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -298,8 +298,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c h1:ezJo/ZSX1elM5SKyNI4YUtGJBdECY78xv/ZmzS9xWaE= -github.com/ipfs/boxo v0.29.2-0.20250416082645-d26d20aaaf7c/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 h1:ByhD/CZId7jjkMLKBi0euA8vEZD6Bma9rysZqSXQuxM= +github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= From 0662522a45d9bef90ff57819edd973526b241ac0 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Thu, 24 Apr 2025 10:11:03 +0200 Subject: [PATCH 06/14] bump boxo --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 2ddc3c12190..e1a1fd79bae 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.24 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 + github.com/ipfs/boxo v0.29.2-0.20250424075053-95aa3f00b161 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.41.1 github.com/multiformats/go-multiaddr v0.15.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 18716e28909..3181abe6bd2 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 h1:ByhD/CZId7jjkMLKBi0euA8vEZD6Bma9rysZqSXQuxM= -github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250424075053-95aa3f00b161 h1:SfqEWWvIMYj+cbmtAFW7/oSrI0FGfs69jro5/w0sLig= +github.com/ipfs/boxo v0.29.2-0.20250424075053-95aa3f00b161/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/go.mod b/go.mod index 10db23aefee..da94a402517 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 + github.com/ipfs/boxo v0.29.2-0.20250424075053-95aa3f00b161 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index cbeb7cef942..749258c45cb 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 h1:ByhD/CZId7jjkMLKBi0euA8vEZD6Bma9rysZqSXQuxM= -github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250424075053-95aa3f00b161 h1:SfqEWWvIMYj+cbmtAFW7/oSrI0FGfs69jro5/w0sLig= +github.com/ipfs/boxo v0.29.2-0.20250424075053-95aa3f00b161/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index f5379818895..592ed59834e 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -118,7 +118,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 // indirect + github.com/ipfs/boxo v0.29.2-0.20250424075053-95aa3f00b161 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cid v0.5.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index c30d121dec4..1267004e971 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -298,8 +298,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01 h1:ByhD/CZId7jjkMLKBi0euA8vEZD6Bma9rysZqSXQuxM= -github.com/ipfs/boxo v0.29.2-0.20250416100703-75ffcd741c01/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= +github.com/ipfs/boxo v0.29.2-0.20250424075053-95aa3f00b161 h1:SfqEWWvIMYj+cbmtAFW7/oSrI0FGfs69jro5/w0sLig= +github.com/ipfs/boxo v0.29.2-0.20250424075053-95aa3f00b161/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= From cc0202258495655250f39e17c172b5b9e71f615d Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Thu, 24 Apr 2025 14:29:31 +0200 Subject: [PATCH 07/14] docs --- docs/changelogs/v0.35.md | 20 ++++++++++++++++++++ docs/config.md | 17 ++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/changelogs/v0.35.md b/docs/changelogs/v0.35.md index bc3a857f8b7..c4af2d2889f 100644 --- a/docs/changelogs/v0.35.md +++ b/docs/changelogs/v0.35.md @@ -17,6 +17,7 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team. - [New `ipfs add` Options](#new-ipfs-add-options) - [Persistent `Import.*` Configuration](#persistent-import-configuration) - [Updated Configuration Profiles](#updated-configuration-profiles) + - [Provide Regression Fix](#provide-regression-fix) - [๐Ÿ“ฆ๏ธ Important dependency updates](#-important-dependency-updates) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -81,6 +82,25 @@ The release updated configuration [profiles](https://github.com/ipfs/kubo/blob/m > [!TIP] > Apply one of CIDv1 test [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) with `ipfs config profile apply test-cid-v1[-wide]`. +#### Provide Regression Fix + +From `kubo` [`v0.33.0`](https://github.com/ipfs/kubo/releases/tag/v0.33.0), +Bitswap stopped advertising newly added and received blocks to the DHT. Since +then `boxo/provider` is responsible for the provide and reprovide logic. Prior +to `v0.35.0`, provides and reprovides were handled together in batches, leading +to delays in initial advertisements (provides). + +Provides and Reprovides now have separate queues, allowing for immediate +provide of new CIDs and optimised batching of reprovides. + +This change introduces a new configuration option for limiting the number of +concurrent provide operations: +[`Provider.WorkerCount`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providerworkercount). + +> [!TIP] +> Users who need to provide large volumes of content immediately should +> consider removing the cap on concurrent provide operations. + #### ๐Ÿ“ฆ๏ธ Important dependency updates - update `ipfs-webui` to [v4.7.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.7.0) diff --git a/docs/config.md b/docs/config.md index 66b72c8c6b5..56ca3c613cb 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1410,9 +1410,24 @@ Type: `duration` ### `Provider.WorkerCount` +Sets the maximum number of _concurrent_ DHT provide operations. DHT reprovides +operations do **not** count against that limit. A value of `0` allows an +unlimited number of provide workers. + +If the [accelerated DHT client](#routingaccelerateddhtclient) is enabled, each +provide operation opens ~20 connections in parallel. With the standard DHT +client (accelerated disabled), each provide opens between 20 and 60 +connections, with at most 10 active at once. Provides complete more quickly +when using the accelerated client. Be mindful of how many simultaneous +connections this setting can generate. + +For nodes without strict connection limits that need to provide large volumes +of content immediately, we recommend enabling the accelerated DHT client and +setting `Provider.WorkerCount` to `0` (unlimited). + Default: `64` -Type: `integer` (non-negative, 0 means unlimited number of workers) +Type: `integer` (non-negative; `0` means unlimited number of workers) ## `Pubsub` From 5d148b8e9c0051c339711bb082bed6c0b2502060 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Tue, 29 Apr 2025 13:26:38 +0200 Subject: [PATCH 08/14] address review --- config/provider.go | 4 +- core/commands/stat.go | 11 ++-- core/commands/stat_provide.go | 52 +++-------------- core/commands/stat_reprovide.go | 100 ++++++++++++++++++++++++++++++++ docs/changelogs/v0.35.md | 18 +++++- docs/config.md | 6 ++ 6 files changed, 138 insertions(+), 53 deletions(-) create mode 100644 core/commands/stat_reprovide.go diff --git a/config/provider.go b/config/provider.go index b62be9499db..d78af924248 100644 --- a/config/provider.go +++ b/config/provider.go @@ -5,6 +5,6 @@ const ( ) type Provider struct { - Strategy string // Which keys to announce - WorkerCount OptionalInteger // Number of concurrent provides allowed, 0 means unlimited + Strategy string // Which keys to announce + WorkerCount *OptionalInteger `json:",omitempty"` // Number of concurrent provides allowed, 0 means unlimited } diff --git a/core/commands/stat.go b/core/commands/stat.go index 2632d6aa257..2b4485a9513 100644 --- a/core/commands/stat.go +++ b/core/commands/stat.go @@ -27,11 +27,12 @@ for your IPFS node.`, }, Subcommands: map[string]*cmds.Command{ - "bw": statBwCmd, - "repo": repoStatCmd, - "bitswap": bitswapStatCmd, - "dht": statDhtCmd, - "provide": statProvideCmd, + "bw": statBwCmd, + "repo": repoStatCmd, + "bitswap": bitswapStatCmd, + "dht": statDhtCmd, + "provide": statProvideCmd, + "reprovide": statReprovideCmd, }, } diff --git a/core/commands/stat_provide.go b/core/commands/stat_provide.go index 1487ae36dde..7f0a70a83dd 100644 --- a/core/commands/stat_provide.go +++ b/core/commands/stat_provide.go @@ -4,28 +4,19 @@ import ( "fmt" "io" "text/tabwriter" - "time" - humanize "github.com/dustin/go-humanize" - "github.com/ipfs/boxo/provider" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/libp2p/go-libp2p-kad-dht/fullrt" - "golang.org/x/exp/constraints" ) -type reprovideStats struct { - provider.ReproviderStats - fullRT bool -} - var statProvideCmd = &cmds.Command{ + Status: cmds.Deprecated, Helptext: cmds.HelpText{ - Tagline: "Returns statistics about the node's (re)provider system.", + Tagline: "Deprecated command to get provider+reprovider statistics. Use 'ipfs stats reprovide' instead.", ShortDescription: ` -Returns statistics about the content the node is advertising. - -This interface is not stable and may change from release to release. +'ipfs stats provide' is depreacted because provide and reprovide operations are now distinct. +This command may be replaced by provide only stats in the future. `, }, Arguments: []cmds.Argument{}, @@ -57,13 +48,13 @@ This interface is not stable and may change from release to release. wtr := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) defer wtr.Flush() - fmt.Fprintf(wtr, "TotalReprovides:\t%s\n", humanNumber(s.TotalReprovides)) - fmt.Fprintf(wtr, "AvgReprovideDuration:\t%s\n", humanDuration(s.AvgReprovideDuration)) + fmt.Fprintf(wtr, "TotalProvides:\t%s\n", humanNumber(s.TotalReprovides)) + fmt.Fprintf(wtr, "AvgProvideDuration:\t%s\n", humanDuration(s.AvgReprovideDuration)) fmt.Fprintf(wtr, "LastReprovideDuration:\t%s\n", humanDuration(s.LastReprovideDuration)) if !s.LastRun.IsZero() { - fmt.Fprintf(wtr, "LastReprovide:\t%s\n", humanTime(s.LastRun)) + fmt.Fprintf(wtr, "LastRun:\t%s\n", humanTime(s.LastRun)) if s.fullRT { - fmt.Fprintf(wtr, "NextReprovide:\t%s\n", humanTime(s.LastRun.Add(s.ReprovideInterval))) + fmt.Fprintf(wtr, "NextRun:\t%s\n", humanTime(s.LastRun.Add(s.ReprovideInterval))) } } return nil @@ -71,30 +62,3 @@ This interface is not stable and may change from release to release. }, Type: reprovideStats{}, } - -func humanDuration(val time.Duration) string { - return val.Truncate(time.Microsecond).String() -} - -func humanTime(val time.Time) string { - return val.Format("2006-01-02 15:04:05") -} - -func humanNumber[T constraints.Float | constraints.Integer](n T) string { - nf := float64(n) - str := humanSI(nf, 0) - fullStr := humanFull(nf, 0) - if str != fullStr { - return fmt.Sprintf("%s\t(%s)", str, fullStr) - } - return str -} - -func humanSI(val float64, decimals int) string { - v, unit := humanize.ComputeSI(val) - return fmt.Sprintf("%s%s", humanFull(v, decimals), unit) -} - -func humanFull(val float64, decimals int) string { - return humanize.CommafWithDigits(val, decimals) -} diff --git a/core/commands/stat_reprovide.go b/core/commands/stat_reprovide.go new file mode 100644 index 00000000000..c8fa237ce26 --- /dev/null +++ b/core/commands/stat_reprovide.go @@ -0,0 +1,100 @@ +package commands + +import ( + "fmt" + "io" + "text/tabwriter" + "time" + + humanize "github.com/dustin/go-humanize" + "github.com/ipfs/boxo/provider" + cmds "github.com/ipfs/go-ipfs-cmds" + "github.com/ipfs/kubo/core/commands/cmdenv" + "github.com/libp2p/go-libp2p-kad-dht/fullrt" + "golang.org/x/exp/constraints" +) + +type reprovideStats struct { + provider.ReproviderStats + fullRT bool +} + +var statReprovideCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Returns statistics about the node's reprovider system.", + ShortDescription: ` +Returns statistics about the content the node is reproviding. + +This interface is not stable and may change from release to release. +`, + }, + Arguments: []cmds.Argument{}, + Options: []cmds.Option{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + if !nd.IsOnline { + return ErrNotOnline + } + + stats, err := nd.Provider.Stat() + if err != nil { + return err + } + _, fullRT := nd.DHTClient.(*fullrt.FullRT) + + if err := res.Emit(reprovideStats{stats, fullRT}); err != nil { + return err + } + + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s reprovideStats) error { + wtr := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) + defer wtr.Flush() + + fmt.Fprintf(wtr, "TotalReprovides:\t%s\n", humanNumber(s.TotalReprovides)) + fmt.Fprintf(wtr, "AvgReprovideDuration:\t%s\n", humanDuration(s.AvgReprovideDuration)) + fmt.Fprintf(wtr, "LastReprovideDuration:\t%s\n", humanDuration(s.LastReprovideDuration)) + if !s.LastRun.IsZero() { + fmt.Fprintf(wtr, "LastReprovide:\t%s\n", humanTime(s.LastRun)) + if s.fullRT { + fmt.Fprintf(wtr, "NextReprovide:\t%s\n", humanTime(s.LastRun.Add(s.ReprovideInterval))) + } + } + return nil + }), + }, + Type: reprovideStats{}, +} + +func humanDuration(val time.Duration) string { + return val.Truncate(time.Microsecond).String() +} + +func humanTime(val time.Time) string { + return val.Format("2006-01-02 15:04:05") +} + +func humanNumber[T constraints.Float | constraints.Integer](n T) string { + nf := float64(n) + str := humanSI(nf, 0) + fullStr := humanFull(nf, 0) + if str != fullStr { + return fmt.Sprintf("%s\t(%s)", str, fullStr) + } + return str +} + +func humanSI(val float64, decimals int) string { + v, unit := humanize.ComputeSI(val) + return fmt.Sprintf("%s%s", humanFull(v, decimals), unit) +} + +func humanFull(val float64, decimals int) string { + return humanize.CommafWithDigits(val, decimals) +} diff --git a/docs/changelogs/v0.35.md b/docs/changelogs/v0.35.md index c4af2d2889f..d61f232b5ec 100644 --- a/docs/changelogs/v0.35.md +++ b/docs/changelogs/v0.35.md @@ -17,7 +17,8 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team. - [New `ipfs add` Options](#new-ipfs-add-options) - [Persistent `Import.*` Configuration](#persistent-import-configuration) - [Updated Configuration Profiles](#updated-configuration-profiles) - - [Provide Regression Fix](#provide-regression-fix) + - [Optimized, dedicated queue for providing fresh CIDs](#optimized-dedicated-queue-for-providing-fresh-cids) + - [Deprecated `ipfs stats provider`](#deprecated-ipfs-stats-provider) - [๐Ÿ“ฆ๏ธ Important dependency updates](#-important-dependency-updates) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -82,7 +83,7 @@ The release updated configuration [profiles](https://github.com/ipfs/kubo/blob/m > [!TIP] > Apply one of CIDv1 test [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) with `ipfs config profile apply test-cid-v1[-wide]`. -#### Provide Regression Fix +#### Optimized, dedicated queue for providing fresh CIDs From `kubo` [`v0.33.0`](https://github.com/ipfs/kubo/releases/tag/v0.33.0), Bitswap stopped advertising newly added and received blocks to the DHT. Since @@ -101,6 +102,19 @@ concurrent provide operations: > Users who need to provide large volumes of content immediately should > consider removing the cap on concurrent provide operations. +##### Deprecated `ipfs stats provider` + +Since the `ipfs stats provider` command was displaying statistics for both +provides and reprovides, this command isn't relevant anymore after separating +the two queues. + +The successor command is `ipfs stats reprovide`, showing the same statistics, +but for reprovides only. + +`ipfs stats provider` still works, but is marked as deprecated and will be +removed in a future release. It provides only statistics about reprovides +(similar to `ipfs stats reprovide`). + #### ๐Ÿ“ฆ๏ธ Important dependency updates - update `ipfs-webui` to [v4.7.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.7.0) diff --git a/docs/config.md b/docs/config.md index 56ca3c613cb..6632290a381 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1408,6 +1408,12 @@ Type: `duration` ## `Provider` +Configuration applied to the initial one-time announcement of fresh CIDs +created with `ipfs add`, `ipfs files`, `ipfs dag import`, `ipfs block|dag put` +commands. + +For periodical DHT reprovide settings, see [`Reprovide.*`](#reprovider). + ### `Provider.WorkerCount` Sets the maximum number of _concurrent_ DHT provide operations. DHT reprovides From 617d5c8118b30987cf096030b712701a72d14dd7 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Tue, 29 Apr 2025 13:39:17 +0200 Subject: [PATCH 09/14] fixing tests --- cmd/ipfs/kubo/daemon.go | 4 ++++ config/config_test.go | 4 ++-- core/commands/commands_test.go | 1 + core/commands/stat_provide.go | 7 ++++--- test/sharness/t0002-docker-image.sh | 4 ++-- test/sharness/t0070-user-config.sh | 8 ++++---- 6 files changed, 17 insertions(+), 11 deletions(-) diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index 7ee5953070c..f732b0773ef 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -485,6 +485,10 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment // This should never happen, but better safe than sorry log.Fatal("Private network does not work with Routing.Type=auto. Update your config to Routing.Type=dht (or none, and do manual peering)") } + if cfg.Provider.Strategy != "" && cfg.Reprovider.Strategy.IsDefault() { + // Provider.Strategy isn't used anywhere. + log.Fatal("Switch to using Reprovider.Strategy instead of Provider.Strategy. Update your config to remove this message.") + } printLibp2pPorts(node) diff --git a/config/config_test.go b/config/config_test.go index d4f38f08662..16573504370 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -134,9 +134,9 @@ func TestCheckKey(t *testing.T) { t.Fatal("Foo.Bar isn't a valid key in the config") } - err = CheckKey("Provider.Strategy") + err = CheckKey("Reprovider.Strategy") if err != nil { - t.Fatalf("%s: %s", err, "Provider.Strategy is a valid key in the config") + t.Fatalf("%s: %s", err, "Reprovider.Strategy is a valid key in the config") } err = CheckKey("Provider.Foo") diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 2dda639f545..d8b4c408380 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -184,6 +184,7 @@ func TestCommands(t *testing.T) { "/stats/bw", "/stats/dht", "/stats/provide", + "/stats/reprovide", "/stats/repo", "/swarm", "/swarm/addrs", diff --git a/core/commands/stat_provide.go b/core/commands/stat_provide.go index 7f0a70a83dd..992cc229f50 100644 --- a/core/commands/stat_provide.go +++ b/core/commands/stat_provide.go @@ -13,10 +13,11 @@ import ( var statProvideCmd = &cmds.Command{ Status: cmds.Deprecated, Helptext: cmds.HelpText{ - Tagline: "Deprecated command to get provider+reprovider statistics. Use 'ipfs stats reprovide' instead.", + Tagline: "Deprecated command, use 'ipfs stats reprovide' instead.", ShortDescription: ` -'ipfs stats provide' is depreacted because provide and reprovide operations are now distinct. -This command may be replaced by provide only stats in the future. +'ipfs stats provide' is depreacted because provide and reprovide operations +are now distinct. This command may be replaced by provide only stats in the +future. `, }, Arguments: []cmds.Argument{}, diff --git a/test/sharness/t0002-docker-image.sh b/test/sharness/t0002-docker-image.sh index 8812c277a74..f894d8f79ab 100755 --- a/test/sharness/t0002-docker-image.sh +++ b/test/sharness/t0002-docker-image.sh @@ -36,7 +36,7 @@ test_expect_success "docker image build succeeds" ' ' test_expect_success "write init scripts" ' - echo "ipfs config Provider.Strategy Bar" > 001.sh && + echo "ipfs config Reprovider.Strategy Bar" > 001.sh && echo "ipfs config Pubsub.Router Qux" > 002.sh && chmod +x 002.sh ' @@ -65,7 +65,7 @@ test_expect_success "check that init scripts were run correctly and in the corre test_expect_success "check that init script configs were applied" ' echo Bar > expected && - docker exec "$DOC_ID" ipfs config Provider.Strategy > actual && + docker exec "$DOC_ID" ipfs config Reprovider.Strategy > actual && test_cmp actual expected && echo Qux > expected && docker exec "$DOC_ID" ipfs config Pubsub.Router > actual && diff --git a/test/sharness/t0070-user-config.sh b/test/sharness/t0070-user-config.sh index 1dc4c0369a0..bc93ca2fa5e 100755 --- a/test/sharness/t0070-user-config.sh +++ b/test/sharness/t0070-user-config.sh @@ -11,12 +11,12 @@ test_description="Test user-provided config values" test_init_ipfs test_expect_success "bootstrap doesn't overwrite user-provided config keys (top-level)" ' - ipfs config Provider.Strategy >previous && - ipfs config Provider.Strategy foo && + ipfs config Reprovider.Strategy >previous && + ipfs config Reprovider.Strategy foo && ipfs bootstrap rm --all && echo "foo" >expected && - ipfs config Provider.Strategy >actual && - ipfs config Provider.Strategy $(cat previous) && + ipfs config Reprovider.Strategy >actual && + ipfs config Reprovider.Strategy $(cat previous) && test_cmp expected actual ' From c65608cd4e85c8b7eb959d2ddb8d522d2293a1d0 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Tue, 29 Apr 2025 14:21:36 +0200 Subject: [PATCH 10/14] adapt sharness tests --- test/sharness/t0002-docker-image.sh | 4 ++-- test/sharness/t0070-user-config.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/sharness/t0002-docker-image.sh b/test/sharness/t0002-docker-image.sh index f894d8f79ab..81bb8d4493f 100755 --- a/test/sharness/t0002-docker-image.sh +++ b/test/sharness/t0002-docker-image.sh @@ -36,7 +36,7 @@ test_expect_success "docker image build succeeds" ' ' test_expect_success "write init scripts" ' - echo "ipfs config Reprovider.Strategy Bar" > 001.sh && + echo "ipfs config Mounts.IPFS Bar" > 001.sh && echo "ipfs config Pubsub.Router Qux" > 002.sh && chmod +x 002.sh ' @@ -65,7 +65,7 @@ test_expect_success "check that init scripts were run correctly and in the corre test_expect_success "check that init script configs were applied" ' echo Bar > expected && - docker exec "$DOC_ID" ipfs config Reprovider.Strategy > actual && + docker exec "$DOC_ID" ipfs config Mounts.IPFS > actual && test_cmp actual expected && echo Qux > expected && docker exec "$DOC_ID" ipfs config Pubsub.Router > actual && diff --git a/test/sharness/t0070-user-config.sh b/test/sharness/t0070-user-config.sh index bc93ca2fa5e..5a8180c7317 100755 --- a/test/sharness/t0070-user-config.sh +++ b/test/sharness/t0070-user-config.sh @@ -11,12 +11,12 @@ test_description="Test user-provided config values" test_init_ipfs test_expect_success "bootstrap doesn't overwrite user-provided config keys (top-level)" ' - ipfs config Reprovider.Strategy >previous && - ipfs config Reprovider.Strategy foo && + ipfs config Identity.PeerID >previous && + ipfs config Identity.PeerID foo && ipfs bootstrap rm --all && echo "foo" >expected && - ipfs config Reprovider.Strategy >actual && - ipfs config Reprovider.Strategy $(cat previous) && + ipfs config Identity.PeerID >actual && + ipfs config Identity.PeerID $(cat previous) && test_cmp expected actual ' From 30bdc1549fb2f4140d1997cf09c89b3b3f9f4e1e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 30 Apr 2025 14:33:27 +0200 Subject: [PATCH 11/14] fix(config): skip unset Provide.Strategy we dont do that for Reprovider.Strategy which is more important, this should reduce user confusion --- config/provider.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/provider.go b/config/provider.go index d78af924248..a1c44859889 100644 --- a/config/provider.go +++ b/config/provider.go @@ -4,7 +4,9 @@ const ( DefaultProviderWorkerCount = 64 ) +// Provider configuration describes how NEW CIDs are announced the moment they are created. +// For periodical reprovide configuration, see Reprovider.* type Provider struct { - Strategy string // Which keys to announce + Strategy *OptionalString `json:",omitempty"` // Unused, you are likely looking for Reprovider.Strategy instead WorkerCount *OptionalInteger `json:",omitempty"` // Number of concurrent provides allowed, 0 means unlimited } From 9a04312d3df9aa24127b29e45694e9179c9ae838 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 30 Apr 2025 14:34:57 +0200 Subject: [PATCH 12/14] docs/chore: mark stat reprovide as experimental --- config/reprovider.go | 2 ++ core/commands/stat_provide.go | 2 +- core/commands/stat_reprovide.go | 5 ++++- docs/changelogs/v0.35.md | 8 +++----- docs/config.md | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/config/reprovider.go b/config/reprovider.go index dae9ae6dee9..3e8a5b476ca 100644 --- a/config/reprovider.go +++ b/config/reprovider.go @@ -7,6 +7,8 @@ const ( DefaultReproviderStrategy = "all" ) +// Reprovider configuration describes how CID from local datastore are periodically re-announced to routing systems. +// For provide behavior of ad-hoc or newly created CIDs and their first-time announcement, see Provider.* type Reprovider struct { Interval *OptionalDuration `json:",omitempty"` // Time period to reprovide locally stored objects to the network Strategy *OptionalString `json:",omitempty"` // Which keys to announce diff --git a/core/commands/stat_provide.go b/core/commands/stat_provide.go index 992cc229f50..ef06d8e2828 100644 --- a/core/commands/stat_provide.go +++ b/core/commands/stat_provide.go @@ -15,7 +15,7 @@ var statProvideCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Deprecated command, use 'ipfs stats reprovide' instead.", ShortDescription: ` -'ipfs stats provide' is depreacted because provide and reprovide operations +'ipfs stats provide' is deprecated because provide and reprovide operations are now distinct. This command may be replaced by provide only stats in the future. `, diff --git a/core/commands/stat_reprovide.go b/core/commands/stat_reprovide.go index c8fa237ce26..dc896747e46 100644 --- a/core/commands/stat_reprovide.go +++ b/core/commands/stat_reprovide.go @@ -20,12 +20,15 @@ type reprovideStats struct { } var statReprovideCmd = &cmds.Command{ + Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Returns statistics about the node's reprovider system.", ShortDescription: ` -Returns statistics about the content the node is reproviding. +Returns statistics about the content the node is reproviding every Reprovider.Interval. This interface is not stable and may change from release to release. + +See https://github.com/ipfs/kubo/blob/master/docs/config.md#reprovider `, }, Arguments: []cmds.Argument{}, diff --git a/docs/changelogs/v0.35.md b/docs/changelogs/v0.35.md index d61f232b5ec..385e32fdf17 100644 --- a/docs/changelogs/v0.35.md +++ b/docs/changelogs/v0.35.md @@ -99,8 +99,7 @@ concurrent provide operations: [`Provider.WorkerCount`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providerworkercount). > [!TIP] -> Users who need to provide large volumes of content immediately should -> consider removing the cap on concurrent provide operations. +> Users who need to provide large volumes of content immediately should consider removing the cap on concurrent provide operations and also set `Routing.AcceleratedDHTClient` to `true`. ##### Deprecated `ipfs stats provider` @@ -111,9 +110,8 @@ the two queues. The successor command is `ipfs stats reprovide`, showing the same statistics, but for reprovides only. -`ipfs stats provider` still works, but is marked as deprecated and will be -removed in a future release. It provides only statistics about reprovides -(similar to `ipfs stats reprovide`). +> [!NOTE] +> `ipfs stats provider` still works, but is marked as deprecated and will be removed in a future release. Be mindful that the command provides only statistics about reprovides (similar to `ipfs stats reprovide`) and not the new provide queue (this will be fixed as a part of wider refactor planned for a future release). #### ๐Ÿ“ฆ๏ธ Important dependency updates diff --git a/docs/config.md b/docs/config.md index 6632290a381..b5e622e825d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1428,7 +1428,7 @@ when using the accelerated client. Be mindful of how many simultaneous connections this setting can generate. For nodes without strict connection limits that need to provide large volumes -of content immediately, we recommend enabling the accelerated DHT client and +of content immediately, we recommend enabling the `Routing.AcceleratedDHTClient` and setting `Provider.WorkerCount` to `0` (unlimited). Default: `64` From 06e66185f8dfb562c6888af17c914116309669ed Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 30 Apr 2025 14:56:39 +0200 Subject: [PATCH 13/14] docs: Provider.Strategy explicitly document it is not used - without this legacy users will have it in their config and be very confused --- cmd/ipfs/kubo/daemon.go | 5 ++--- core/commands/stat_reprovide.go | 5 +++-- docs/config.md | 6 ++++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index f732b0773ef..c237beb5772 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -485,9 +485,8 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment // This should never happen, but better safe than sorry log.Fatal("Private network does not work with Routing.Type=auto. Update your config to Routing.Type=dht (or none, and do manual peering)") } - if cfg.Provider.Strategy != "" && cfg.Reprovider.Strategy.IsDefault() { - // Provider.Strategy isn't used anywhere. - log.Fatal("Switch to using Reprovider.Strategy instead of Provider.Strategy. Update your config to remove this message.") + if cfg.Provider.Strategy.WithDefault("") != "" && cfg.Reprovider.Strategy.IsDefault() { + log.Fatal("Invalid config. Remove unsued Provider.Strategy and set Reprovider.Strategy instead. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy") } printLibp2pPorts(node) diff --git a/core/commands/stat_reprovide.go b/core/commands/stat_reprovide.go index dc896747e46..10dbc727d38 100644 --- a/core/commands/stat_reprovide.go +++ b/core/commands/stat_reprovide.go @@ -24,11 +24,12 @@ var statReprovideCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Returns statistics about the node's reprovider system.", ShortDescription: ` -Returns statistics about the content the node is reproviding every Reprovider.Interval. +Returns statistics about the content the node is reproviding every +Reprovider.Interval according to Reprovider.Strategy: +https://github.com/ipfs/kubo/blob/master/docs/config.md#reprovider This interface is not stable and may change from release to release. -See https://github.com/ipfs/kubo/blob/master/docs/config.md#reprovider `, }, Arguments: []cmds.Argument{}, diff --git a/docs/config.md b/docs/config.md index b5e622e825d..af308c4c08a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -106,6 +106,7 @@ config file at runtime. - [`Pinning.RemoteServices: Policies.MFS.PinName`](#pinningremoteservices-policiesmfspinname) - [`Pinning.RemoteServices: Policies.MFS.RepinInterval`](#pinningremoteservices-policiesmfsrepininterval) - [`Provider`](#provider) + - [`Provider.Strategy`](#providerstrategy) - [`Provider.WorkerCount`](#providerworkercount) - [`Pubsub`](#pubsub) - [`Pubsub.Enabled`](#pubsubenabled) @@ -209,6 +210,7 @@ config file at runtime. - [`announce-on` profile](#announce-on-profile) - [`legacy-cid-v0` profile](#legacy-cid-v0-profile) - [`test-cid-v1` profile](#test-cid-v1-profile) + - [`test-cid-v1-wide` profile](#test-cid-v1-wide-profile) - [Types](#types) - [`flag`](#flag) - [`priority`](#priority) @@ -1414,6 +1416,10 @@ commands. For periodical DHT reprovide settings, see [`Reprovide.*`](#reprovider). +### `Provider.Strategy` + +Legacy, not used at the moment, see [`Reprovider.Strategy`](#reproviderstrategy) instead. + ### `Provider.WorkerCount` Sets the maximum number of _concurrent_ DHT provide operations. DHT reprovides From 74cf6afd42383b9c596aef88bf3a83914ccd29a2 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 30 Apr 2025 15:27:39 +0200 Subject: [PATCH 14/14] chore: fix typo --- cmd/ipfs/kubo/daemon.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index c237beb5772..30e805186c1 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -486,7 +486,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment log.Fatal("Private network does not work with Routing.Type=auto. Update your config to Routing.Type=dht (or none, and do manual peering)") } if cfg.Provider.Strategy.WithDefault("") != "" && cfg.Reprovider.Strategy.IsDefault() { - log.Fatal("Invalid config. Remove unsued Provider.Strategy and set Reprovider.Strategy instead. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy") + log.Fatal("Invalid config. Remove unused Provider.Strategy and set Reprovider.Strategy instead. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy") } printLibp2pPorts(node)