Skip to content

Commit

Permalink
opt: support offline compute commp
Browse files Browse the repository at this point in the history
  • Loading branch information
simlecode committed Oct 8, 2023
1 parent f1dbb57 commit fea1825
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 48 deletions.
43 changes: 1 addition & 42 deletions client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ import (

"github.com/filecoin-project/go-address"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-commp-utils/ffiwrapper"
"github.com/filecoin-project/go-commp-utils/writer"
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
"github.com/filecoin-project/go-fil-markets/discovery"
Expand Down Expand Up @@ -1332,47 +1331,7 @@ func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Addre
}

func (a *API) ClientCalcCommP(ctx context.Context, inpath string) (*types.CommPRet, error) {
// Hard-code the sector type to 32GiBV1_1, because:
// - ffiwrapper.GeneratePieceCIDFromFile requires a RegisteredSealProof
// - commP itself is sector-size independent, with rather low probability of that changing
// ( note how the final rust call is identical for every RegSP type )
// https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/v5.0.0/src/seal.rs#L1040-L1050
//
// IF/WHEN this changes in the future we will have to be able to calculate
// "old style" commP, and thus will need to introduce a version switch or similar
arbitraryProofType := abi.RegisteredSealProof_StackedDrg32GiBV1_1

rdr, err := os.Open(inpath)
if err != nil {
return nil, err
}
defer rdr.Close() //nolint:errcheck

stat, err := rdr.Stat()
if err != nil {
return nil, err
}

// check that the data is a car file; if it's not, retrieval won't work
_, err = car.ReadHeader(bufio.NewReader(rdr))
if err != nil {
return nil, fmt.Errorf("not a car file: %w", err)
}

if _, err := rdr.Seek(0, io.SeekStart); err != nil {
return nil, fmt.Errorf("seek to start: %w", err)
}

pieceReader, pieceSize := padreader.New(rdr, uint64(stat.Size()))
commP, err := ffiwrapper.GeneratePieceCIDFromFile(arbitraryProofType, pieceReader, pieceSize)
if err != nil {
return nil, fmt.Errorf("computing commP failed: %w", err)
}

return &types.CommPRet{
Root: commP,
Size: pieceSize,
}, nil
return utils.CalcCommP(ctx, inpath)
}

type lenWriter int64
Expand Down
8 changes: 2 additions & 6 deletions cmd/droplet-client/data.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/filecoin-project/venus/venus-shared/types"
"github.com/filecoin-project/venus/venus-shared/types/market/client"
cli2 "github.com/ipfs-force-community/droplet/v2/cli"
"github.com/ipfs-force-community/droplet/v2/utils"
)

var dataCmd = &cli.Command{
Expand Down Expand Up @@ -205,18 +206,13 @@ var dataCommPCmd = &cli.Command{
&cli2.CidBaseFlag,
},
Action: func(cctx *cli.Context) error {
api, closer, err := cli2.NewMarketClientNode(cctx)
if err != nil {
return err
}
defer closer()
ctx := cli2.ReqContext(cctx)

if cctx.Args().Len() != 1 {
return fmt.Errorf("usage: commP <inputPath>")
}

ret, err := api.ClientCalcCommP(ctx, cctx.Args().Get(0))
ret, err := utils.CalcCommP(ctx, cctx.Args().Get(0))
if err != nil {
return err
}
Expand Down
59 changes: 59 additions & 0 deletions utils/commp.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package utils

import (
"bufio"
"context"
"fmt"
"io"
"os"

"github.com/filecoin-project/go-commp-utils/ffiwrapper"
"github.com/filecoin-project/go-padreader"
"github.com/filecoin-project/go-state-types/abi"
types "github.com/filecoin-project/venus/venus-shared/types/market/client"
"github.com/ipld/go-car"
)

func CalcCommP(ctx context.Context, inpath string) (*types.CommPRet, error) {
// Hard-code the sector type to 32GiBV1_1, because:
// - ffiwrapper.GeneratePieceCIDFromFile requires a RegisteredSealProof
// - commP itself is sector-size independent, with rather low probability of that changing
// ( note how the final rust call is identical for every RegSP type )
// https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/v5.0.0/src/seal.rs#L1040-L1050
//
// IF/WHEN this changes in the future we will have to be able to calculate
// "old style" commP, and thus will need to introduce a version switch or similar
arbitraryProofType := abi.RegisteredSealProof_StackedDrg32GiBV1_1

rdr, err := os.Open(inpath)
if err != nil {
return nil, err
}
defer rdr.Close() //nolint:errcheck

stat, err := rdr.Stat()
if err != nil {
return nil, err
}

// check that the data is a car file; if it's not, retrieval won't work
_, err = car.ReadHeader(bufio.NewReader(rdr))
if err != nil {
return nil, fmt.Errorf("not a car file: %w", err)
}

if _, err := rdr.Seek(0, io.SeekStart); err != nil {
return nil, fmt.Errorf("seek to start: %w", err)
}

pieceReader, pieceSize := padreader.New(rdr, uint64(stat.Size()))
commP, err := ffiwrapper.GeneratePieceCIDFromFile(arbitraryProofType, pieceReader, pieceSize)
if err != nil {
return nil, fmt.Errorf("computing commP failed: %w", err)
}

return &types.CommPRet{
Root: commP,
Size: pieceSize,
}, nil
}

0 comments on commit fea1825

Please sign in to comment.