diff --git a/pinning/pinner/dspinner/pin.go b/pinning/pinner/dspinner/pin.go index 350d42fa6..47ec7fea5 100644 --- a/pinning/pinner/dspinner/pin.go +++ b/pinning/pinner/dspinner/pin.go @@ -665,16 +665,16 @@ func (p *pinner) loadPin(ctx context.Context, pid string) (*pin, error) { } // DirectKeys returns a slice containing the directly pinned keys -func (p *pinner) DirectKeys(ctx context.Context) <-chan ipfspinner.StreamedPin { - return p.streamIndex(ctx, p.cidDIndex) +func (p *pinner) DirectKeys(ctx context.Context, detailed bool) <-chan ipfspinner.StreamedPin { + return p.streamIndex(ctx, p.cidDIndex, detailed) } // RecursiveKeys returns a slice containing the recursively pinned keys -func (p *pinner) RecursiveKeys(ctx context.Context) <-chan ipfspinner.StreamedPin { - return p.streamIndex(ctx, p.cidRIndex) +func (p *pinner) RecursiveKeys(ctx context.Context, detailed bool) <-chan ipfspinner.StreamedPin { + return p.streamIndex(ctx, p.cidRIndex, detailed) } -func (p *pinner) streamIndex(ctx context.Context, index dsindex.Indexer) <-chan ipfspinner.StreamedPin { +func (p *pinner) streamIndex(ctx context.Context, index dsindex.Indexer, detailed bool) <-chan ipfspinner.StreamedPin { out := make(chan ipfspinner.StreamedPin) go func() { @@ -692,21 +692,26 @@ func (p *pinner) streamIndex(ctx context.Context, index dsindex.Indexer) <-chan return false } - pp, err := p.loadPin(ctx, value) - if err != nil { - out <- ipfspinner.StreamedPin{Err: err} - return false + var pin ipfspinner.Pinned + if detailed { + pp, err := p.loadPin(ctx, value) + if err != nil { + out <- ipfspinner.StreamedPin{Err: err} + return false + } + + pin.Key = pp.Cid + pin.Mode = pp.Mode + pin.Name = pp.Name + } else { + pin.Key = c } if !cidSet.Has(c) { select { case <-ctx.Done(): return false - case out <- ipfspinner.StreamedPin{Pin: ipfspinner.Pinned{ - Key: pp.Cid, - Mode: pp.Mode, - Name: pp.Name, - }}: + case out <- ipfspinner.StreamedPin{Pin: pin}: } cidSet.Add(c) } @@ -722,7 +727,7 @@ func (p *pinner) streamIndex(ctx context.Context, index dsindex.Indexer) <-chan // InternalPins returns all cids kept pinned for the internal state of the // pinner -func (p *pinner) InternalPins(ctx context.Context) <-chan ipfspinner.StreamedPin { +func (p *pinner) InternalPins(ctx context.Context, detailed bool) <-chan ipfspinner.StreamedPin { c := make(chan ipfspinner.StreamedPin) close(c) return c diff --git a/pinning/pinner/dspinner/pin_test.go b/pinning/pinner/dspinner/pin_test.go index 0b1d15d81..25dd4c009 100644 --- a/pinning/pinner/dspinner/pin_test.go +++ b/pinning/pinner/dspinner/pin_test.go @@ -11,6 +11,7 @@ import ( bs "github.com/ipfs/boxo/blockservice" mdag "github.com/ipfs/boxo/ipld/merkledag" + "github.com/stretchr/testify/require" cid "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -210,7 +211,7 @@ func TestPinnerBasic(t *testing.T) { return pins } - pins := allPins(p.RecursiveKeys(ctx)) + pins := allPins(p.RecursiveKeys(ctx, true)) if len(pins) != 2 { t.Error("expected 2 recursive pins") } @@ -255,7 +256,7 @@ func TestPinnerBasic(t *testing.T) { } } - pins = allPins(p.DirectKeys(ctx)) + pins = allPins(p.DirectKeys(ctx, false)) if len(pins) != 1 { t.Error("expected 1 direct pin") } @@ -263,7 +264,7 @@ func TestPinnerBasic(t *testing.T) { t.Error("wrong direct pin") } - pins = allPins(p.InternalPins(ctx)) + pins = allPins(p.InternalPins(ctx, false)) if len(pins) != 0 { t.Error("should not have internal keys") } @@ -1351,3 +1352,42 @@ func verifyIndexValue(ctx context.Context, pinner *pinner, cidKey, expectedPid s } return nil } + +func BenchmarkDetails(b *testing.B) { + for count := 128; count <= 16386; count <<= 1 { + b.Run(fmt.Sprint("Keys-NoDetails-", count), func(b *testing.B) { + benchmarkDetails(b, count, false) + }) + + b.Run(fmt.Sprint("Keys-Details-", count), func(b *testing.B) { + benchmarkDetails(b, count, true) + }) + } +} + +func benchmarkDetails(b *testing.B, count int, details bool) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore, dserv := makeStore() + pinner, err := New(ctx, dstore, dserv) + require.NoError(b, err) + nodes := makeNodes(count, dserv) + + // Pin all the nodes one at a time. + for j := range nodes { + err := pinner.Pin(ctx, nodes[j], true, "") + require.NoError(b, err) + + err = pinner.Flush(ctx) + require.NoError(b, err) + } + + // Reset the timer and execute actual benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + for val := range pinner.RecursiveKeys(ctx, details) { + require.NoError(b, val.Err) + } + } +} diff --git a/pinning/pinner/pin.go b/pinning/pinner/pin.go index f6b0eb6a6..1670735d7 100644 --- a/pinning/pinner/pin.go +++ b/pinning/pinner/pin.go @@ -120,14 +120,14 @@ type Pinner interface { Flush(ctx context.Context) error // DirectKeys returns all directly pinned cids - DirectKeys(ctx context.Context) <-chan StreamedPin + DirectKeys(ctx context.Context, detailed bool) <-chan StreamedPin // RecursiveKeys returns all recursively pinned cids - RecursiveKeys(ctx context.Context) <-chan StreamedPin + RecursiveKeys(ctx context.Context, detailed bool) <-chan StreamedPin // InternalPins returns all cids kept pinned for the internal state of the // pinner - InternalPins(ctx context.Context) <-chan StreamedPin + InternalPins(ctx context.Context, detailed bool) <-chan StreamedPin } // Pinned represents CID which has been pinned with a pinning strategy. diff --git a/provider/provider.go b/provider/provider.go index 6035fd92d..530469fc1 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -80,7 +80,7 @@ func pinSet(ctx context.Context, pinning pin.Pinner, fetchConfig fetcher.Factory defer cancel() defer close(set.New) - for sc := range pinning.DirectKeys(ctx) { + for sc := range pinning.DirectKeys(ctx, false) { if sc.Err != nil { logR.Errorf("reprovide direct pins: %s", sc.Err) return @@ -89,7 +89,7 @@ func pinSet(ctx context.Context, pinning pin.Pinner, fetchConfig fetcher.Factory } session := fetchConfig.NewSession(ctx) - for sc := range pinning.RecursiveKeys(ctx) { + for sc := range pinning.RecursiveKeys(ctx, false) { if sc.Err != nil { logR.Errorf("reprovide recursive pins: %s", sc.Err) return