Update go-graphsync v0.8.0

.circleci/config.yml

@@ -27,7 +27,7 @@ commands:
           condition: << parameters.linux >>
           steps:
             - run: sudo apt-get update
-            - run: sudo apt-get install ocl-icd-opencl-dev
+            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
       - run: git submodule sync
       - run: git submodule update --init
   build-all:

CHANGELOG.md

@@ -1,5 +1,19 @@
 # go-fil-markets changelog
 
+# go-fil-markets v1.5.0
+
+Adds Dynamic Retrieval Pricing
+
+- github.com/filecoin-project/go-fil-markets:
+  - Dynamic Retrieval Pricing (#542) ([filecoin-project/go-fil-markets#542](https://github.com/filecoin-project/go-fil-markets/pull/542))
+
+Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| Aarsh Shah | 1 | +1224/-130 | 17 |
+| dirkmc | 1 | +18/-0 | 1 |
+
 # go-fil-markets v1.4.0
 
 Generating Changelog for github.com/filecoin-project/go-fil-markets v1.3.0..3511c5617142836e369e31890f544bdd574d970f

go.mod

@@ -6,9 +6,9 @@ require (
 	github.com/filecoin-project/go-address v0.0.3
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434
-	github.com/filecoin-project/go-data-transfer v1.6.0
+	github.com/filecoin-project/go-data-transfer v1.6.1-0.20210601213510-73edd813d563
 	github.com/filecoin-project/go-ds-versioning v0.1.0
-	github.com/filecoin-project/go-multistore v0.0.3
+	github.com/filecoin-project/go-multistore v0.0.4-0.20210601185713-428a2691a567
 	github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20
 	github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc
 	github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe
@@ -17,11 +17,11 @@ require (
 	github.com/filecoin-project/specs-actors/v2 v2.3.2
 	github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1
 	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e
-	github.com/ipfs/go-block-format v0.0.2
+	github.com/ipfs/go-block-format v0.0.3
 	github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834
 	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-datastore v0.4.5
-	github.com/ipfs/go-graphsync v0.6.1
+	github.com/ipfs/go-graphsync v0.8.0
 	github.com/ipfs/go-ipfs-blockstore v1.0.3
 	github.com/ipfs/go-ipfs-blocksutil v0.0.1
 	github.com/ipfs/go-ipfs-chunker v0.0.5
@@ -32,15 +32,17 @@ require (
 	github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4
 	github.com/ipfs/go-merkledag v0.3.2
 	github.com/ipfs/go-unixfs v0.2.4
-	github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d
-	github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018
+	github.com/ipld/go-car v0.3.1-0.20210601190600-f512dac51e8e
+	github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
 	github.com/jpillora/backoff v1.0.0
-	github.com/libp2p/go-libp2p v0.12.0
-	github.com/libp2p/go-libp2p-core v0.7.0
+	github.com/libp2p/go-libp2p v0.13.0
+	github.com/libp2p/go-libp2p-core v0.8.5
+	github.com/minio/sha256-simd v1.0.0
 	github.com/multiformats/go-multiaddr v0.3.1
 	github.com/multiformats/go-multibase v0.0.3
-	github.com/stretchr/testify v1.6.1
+	github.com/multiformats/go-multihash v0.0.15
+	github.com/stretchr/testify v1.7.0
 	github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2
 	golang.org/x/exp v0.0.0-20200207192155-f17229e696bd
 	golang.org/x/net v0.0.0-20201021035429-f5854403a974

retrievalmarket/common.go

@@ -14,7 +14,7 @@ import (
 func DecodeNode(defnode *cbg.Deferred) (ipld.Node, error) {
 	reader := bytes.NewReader(defnode.Raw)
 	nb := basicnode.Prototype.Any.NewBuilder()
-	err := dagcbor.Decoder(nb, reader)
+	err := dagcbor.Decode(nb, reader)
 	if err != nil {
 		return nil, err
 	}

retrievalmarket/impl/client_test.go

@@ -393,7 +393,7 @@ func TestMigrations(t *testing.T) {
 	selfPeer := tut.GeneratePeers(1)[0]
 
 	allSelectorBuf := new(bytes.Buffer)
-	err = dagcbor.Encoder(shared.AllSelector(), allSelectorBuf)
+	err = dagcbor.Encode(shared.AllSelector(), allSelectorBuf)
 	require.NoError(t, err)
 	allSelectorBytes := allSelectorBuf.Bytes()
 

retrievalmarket/impl/dtutils/dtutils.go

@@ -162,7 +162,7 @@ type StoreGetter interface {
 // StoreConfigurableTransport defines the methods needed to
 // configure a data transfer transport use a unique store for a given request
 type StoreConfigurableTransport interface {
-	UseStore(datatransfer.ChannelID, ipld.Loader, ipld.Storer) error
+	UseStore(datatransfer.ChannelID, ipld.LinkSystem) error
 }
 
 // TransportConfigurer configurers the graphsync transport to use a custom blockstore per deal
@@ -185,7 +185,7 @@ func TransportConfigurer(thisPeer peer.ID, storeGetter StoreGetter) datatransfer
 		if store == nil {
 			return
 		}
-		err = gsTransport.UseStore(channelID, store.Loader, store.Storer)
+		err = gsTransport.UseStore(channelID, store.LinkSystem)
 		if err != nil {
 			log.Errorf("attempting to configure data store: %s", err)
 		}

retrievalmarket/impl/dtutils/dtutils_test.go

@@ -493,16 +493,14 @@ func (ft *fakeTransport) Shutdown(context.Context) error {
 
 type fakeGsTransport struct {
 	datatransfer.Transport
-	lastChannelID datatransfer.ChannelID
-	lastLoader    ipld.Loader
-	lastStorer    ipld.Storer
-	called        bool
+	lastChannelID  datatransfer.ChannelID
+	lastLinkSystem ipld.LinkSystem
+	called         bool
 }
 
-func (fgt *fakeGsTransport) UseStore(channelID datatransfer.ChannelID, loader ipld.Loader, storer ipld.Storer) error {
+func (fgt *fakeGsTransport) UseStore(channelID datatransfer.ChannelID, lsys ipld.LinkSystem) error {
 	fgt.lastChannelID = channelID
-	fgt.lastLoader = loader
-	fgt.lastStorer = storer
+	fgt.lastLinkSystem = lsys
 	fgt.called = true
 	return nil
 }

retrievalmarket/impl/integration_test.go

@@ -109,7 +109,7 @@ func requireSetupTestClientAndProvider(ctx context.Context, t *testing.T, payChA
 		AddFundsCID:    cids[1],
 	})
 
-	gs1 := graphsyncimpl.New(ctx, network.NewFromLibp2pHost(testData.Host1), testData.Loader1, testData.Storer1)
+	gs1 := graphsyncimpl.New(ctx, network.NewFromLibp2pHost(testData.Host1), testData.LinkSystem1)
 	dtTransport1 := dtgstransport.NewTransport(testData.Host1.ID(), gs1)
 	dt1, err := dtimpl.NewDataTransfer(testData.DTStore1, testData.DTTmpDir1, testData.DTNet1, dtTransport1)
 	require.NoError(t, err)
@@ -149,7 +149,7 @@ func requireSetupTestClientAndProvider(ctx context.Context, t *testing.T, payChA
 
 	paymentAddress := address.TestAddress2
 
-	gs2 := graphsyncimpl.New(ctx, network.NewFromLibp2pHost(testData.Host2), testData.Loader2, testData.Storer2)
+	gs2 := graphsyncimpl.New(ctx, network.NewFromLibp2pHost(testData.Host2), testData.LinkSystem2)
 	dtTransport2 := dtgstransport.NewTransport(testData.Host2.ID(), gs2)
 	dt2, err := dtimpl.NewDataTransfer(testData.DTStore2, testData.DTTmpDir2, testData.DTNet2, dtTransport2)
 	require.NoError(t, err)
@@ -633,7 +633,7 @@ func setupClient(
 		ChannelAvailableFunds:  channelAvailableFunds,
 	})
 
-	gs1 := graphsyncimpl.New(ctx, network.NewFromLibp2pHost(testData.Host1), testData.Loader1, testData.Storer1)
+	gs1 := graphsyncimpl.New(ctx, network.NewFromLibp2pHost(testData.Host1), testData.LinkSystem1)
 	dtTransport1 := dtgstransport.NewTransport(testData.Host1.ID(), gs1)
 	dt1, err := dtimpl.NewDataTransfer(testData.DTStore1, testData.DTTmpDir1, testData.DTNet1, dtTransport1)
 	require.NoError(t, err)
@@ -670,7 +670,7 @@ func setupProvider(
 	pieceStore.ExpectCID(payloadCID, cidInfo)
 	pieceStore.ExpectPiece(expectedPiece, pieceInfo)
 
-	gs2 := graphsyncimpl.New(ctx, network.NewFromLibp2pHost(testData.Host2), testData.Loader2, testData.Storer2)
+	gs2 := graphsyncimpl.New(ctx, network.NewFromLibp2pHost(testData.Host2), testData.LinkSystem2)
 	dtTransport2 := dtgstransport.NewTransport(testData.Host2.ID(), gs2)
 	dt2, err := dtimpl.NewDataTransfer(testData.DTStore2, testData.DTTmpDir2, testData.DTNet2, dtTransport2)
 	require.NoError(t, err)

retrievalmarket/impl/provider_test.go

@@ -1054,7 +1054,7 @@ func TestProviderMigrations(t *testing.T) {
 	offsets := make([]abi.PaddedPieceSize, numDeals)
 	lengths := make([]abi.PaddedPieceSize, numDeals)
 	allSelectorBuf := new(bytes.Buffer)
-	err = dagcbor.Encoder(shared.AllSelector(), allSelectorBuf)
+	err = dagcbor.Encode(shared.AllSelector(), allSelectorBuf)
 	require.NoError(t, err)
 	allSelectorBytes := allSelectorBuf.Bytes()
 

retrievalmarket/impl/requestvalidation/requestvalidation.go

@@ -28,7 +28,7 @@ var askTimeout = 5 * time.Second
 
 func init() {
 	buf := new(bytes.Buffer)
-	_ = dagcbor.Encoder(shared.AllSelector(), buf)
+	_ = dagcbor.Encode(shared.AllSelector(), buf)
 	allSelectorBytes = buf.Bytes()
 }
 
@@ -105,7 +105,7 @@ func (rv *ProviderRequestValidator) validatePull(isRestart bool, receiver peer.I
 
 	// Check the proposal selector matches
 	buf := new(bytes.Buffer)
-	err := dagcbor.Encoder(selector, buf)
+	err := dagcbor.Encode(selector, buf)
 	if err != nil {
 		return nil, err
 	}

retrievalmarket/types.go

@@ -296,7 +296,7 @@ func NewParamsV1(pricePerByte abi.TokenAmount, paymentInterval uint64, paymentIn
 		return Params{}, xerrors.New("selector required for NewParamsV1")
 	}
 
-	err := dagcbor.Encoder(sel, &buffer)
+	err := dagcbor.Encode(sel, &buffer)
 	if err != nil {
 		return Params{}, xerrors.Errorf("error encoding selector: %w", err)
 	}

retrievalmarket/types_test.go

@@ -37,7 +37,7 @@ func TestParamsMarshalUnmarshal(t *testing.T) {
 	assert.Equal(t, params, *unmarshalled)
 
 	nb := basicnode.Prototype.Any.NewBuilder()
-	err = dagcbor.Decoder(nb, bytes.NewBuffer(unmarshalled.Selector.Raw))
+	err = dagcbor.Decode(nb, bytes.NewBuffer(unmarshalled.Selector.Raw))
 	assert.NoError(t, err)
 	sel := nb.Build()
 	assert.Equal(t, sel, allSelector)

shared_testutil/makecid.go

@@ -0,0 +1,40 @@
+package shared_testutil
+
+import (
+	"errors"
+
+	"github.com/ipfs/go-cid"
+	"github.com/minio/sha256-simd"
+	mh "github.com/multiformats/go-multihash"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+// make a cid directly with a given input and prefix
+func MakeCID(input string, prefix *cid.Prefix) cid.Cid {
+	data := []byte(input)
+	if prefix == nil {
+		c, err := abi.CidBuilder.Sum(data)
+		if err != nil {
+			panic(err)
+		}
+		return c
+	}
+	c, err := prefix.Sum(data)
+	switch {
+	case errors.Is(err, mh.ErrSumNotSupported):
+		// multihash library doesn't support this hash function.
+		// just fake it.
+	case err == nil:
+		return c
+	default:
+		panic(err)
+	}
+
+	sum := sha256.Sum256(data)
+	hash, err := mh.Encode(sum[:], prefix.MhType)
+	if err != nil {
+		panic(err)
+	}
+	return cid.NewCidV1(prefix.Codec, hash)
+}

shared_testutil/mocknet.go

@@ -2,7 +2,6 @@ package shared_testutil
 
 import (
 	"bytes"
-	"errors"
 	"io"
 	"io/ioutil"
 	"os"
@@ -11,11 +10,11 @@ import (
 	"runtime"
 	"testing"
 
-	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 	dss "github.com/ipfs/go-datastore/sync"
+	"github.com/ipfs/go-graphsync/storeutil"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
 	chunk "github.com/ipfs/go-ipfs-chunker"
 	offline "github.com/ipfs/go-ipfs-exchange-offline"
@@ -52,10 +51,8 @@ type Libp2pTestData struct {
 	DTStore2    datastore.Batching
 	DTTmpDir1   string
 	DTTmpDir2   string
-	Loader1     ipld.Loader
-	Loader2     ipld.Loader
-	Storer1     ipld.Storer
-	Storer2     ipld.Storer
+	LinkSystem1 ipld.LinkSystem
+	LinkSystem2 ipld.LinkSystem
 	Host1       host.Host
 	Host2       host.Host
 	OrigBytes   []byte
@@ -66,38 +63,7 @@ type Libp2pTestData struct {
 func NewLibp2pTestData(ctx context.Context, t *testing.T) *Libp2pTestData {
 	testData := &Libp2pTestData{}
 	testData.Ctx = ctx
-	makeLoader := func(bs bstore.Blockstore) ipld.Loader {
-		return func(lnk ipld.Link, lnkCtx ipld.LinkContext) (io.Reader, error) {
-			c, ok := lnk.(cidlink.Link)
-			if !ok {
-				return nil, errors.New("incorrect Link Type")
-			}
-			// read block from one store
-			block, err := bs.Get(c.Cid)
-			if err != nil {
-				return nil, err
-			}
-			return bytes.NewReader(block.RawData()), nil
-		}
-	}
 
-	makeStorer := func(bs bstore.Blockstore) ipld.Storer {
-		return func(lnkCtx ipld.LinkContext) (io.Writer, ipld.StoreCommitter, error) {
-			var buf bytes.Buffer
-			var committer ipld.StoreCommitter = func(lnk ipld.Link) error {
-				c, ok := lnk.(cidlink.Link)
-				if !ok {
-					return errors.New("incorrect Link Type")
-				}
-				block, err := blocks.NewBlockWithCid(buf.Bytes(), c.Cid)
-				if err != nil {
-					return err
-				}
-				return bs.Put(block)
-			}
-			return &buf, committer, nil
-		}
-	}
 	var err error
 
 	testData.Ds1 = dss.MutexWrap(datastore.NewMapDatastore())
@@ -115,13 +81,11 @@ func NewLibp2pTestData(ctx context.Context, t *testing.T) *Libp2pTestData {
 	testData.DagService1 = merkledag.NewDAGService(blockservice.New(testData.Bs1, offline.Exchange(testData.Bs1)))
 	testData.DagService2 = merkledag.NewDAGService(blockservice.New(testData.Bs2, offline.Exchange(testData.Bs2)))
 
-	// setup an IPLD loader/storer for bstore 1
-	testData.Loader1 = makeLoader(testData.Bs1)
-	testData.Storer1 = makeStorer(testData.Bs1)
+	// setup an IPLD link system for bstore 1
+	testData.LinkSystem1 = storeutil.LinkSystemForBlockstore(testData.Bs1)
 
-	// setup an IPLD loader/storer for bstore 2
-	testData.Loader2 = makeLoader(testData.Bs2)
-	testData.Storer2 = makeStorer(testData.Bs2)
+	// setup an IPLD link system for bstore 2
+	testData.LinkSystem2 = storeutil.LinkSystemForBlockstore(testData.Bs2)
 
 	mn := mocknet.New(ctx)