Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

integrate direct data onboarding (nv22) #1847

Merged
merged 15 commits into from
Mar 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ orbs:
executors:
golang:
docker:
- image: cimg/go:1.20.7
- image: cimg/go:1.21.7
resource_class: 2xlarge
ubuntu:
docker:
- image: ubuntu:22.04
- image: ubuntu:default

commands:
install-deps:
Expand Down Expand Up @@ -87,7 +87,7 @@ jobs:
lid-docker-compose:
description: 'Run LID integration tests'
machine:
image: ubuntu-2204:2023.07.2
image: default
resource_class: xlarge
steps:
- checkout
Expand Down Expand Up @@ -132,7 +132,7 @@ jobs:
# local index directory tests so it may be worth refactoring so that it's
# only included for local index directory
- setup_remote_docker:
version: 20.10.23
version: default
docker_layer_caching: true
- run:
name: go test
Expand All @@ -154,7 +154,7 @@ jobs:
linux: false
darwin: true
- golang/install:
version: "1.20.7"
version: "1.21.7"
- run:
name: Install pkg-config
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
Expand Down Expand Up @@ -253,7 +253,7 @@ jobs:
default: golang
golangci-lint-version:
type: string
default: 1.52.2
default: 1.56.2
concurrency:
type: string
default: '2'
Expand Down Expand Up @@ -328,10 +328,10 @@ workflows:
suite: itest-retrieval
target: "./itests/graphsync_retrieval_test.go"

#- test:
#name: test-itest-direct_deal
#suite: itest-direct_deal
#target: "./itests/ddo_test.go"
- test:
name: test-itest-direct_deal
suite: itest-direct_deal
target: "./itests/ddo_test.go"

- test:
name: test-all
Expand Down
40 changes: 39 additions & 1 deletion api/api_full.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,45 @@ type MarketBalance struct {

type MarketDeal struct {
Proposal market.DealProposal
State market.DealState
State MarketDealState
}

type MarketDealState struct {
SectorStartEpoch abi.ChainEpoch // -1 if not yet included in proven sector
LastUpdatedEpoch abi.ChainEpoch // -1 if deal state never updated
SlashEpoch abi.ChainEpoch // -1 if deal never slashed
}

func MakeDealState(mds market.DealState) MarketDealState {
return MarketDealState{
SectorStartEpoch: mds.SectorStartEpoch(),
LastUpdatedEpoch: mds.LastUpdatedEpoch(),
SlashEpoch: mds.SlashEpoch(),
}
}

type mstate struct {
s MarketDealState
}

func (m mstate) SectorStartEpoch() abi.ChainEpoch {
return m.s.SectorStartEpoch
}

func (m mstate) LastUpdatedEpoch() abi.ChainEpoch {
return m.s.LastUpdatedEpoch
}

func (m mstate) SlashEpoch() abi.ChainEpoch {
return m.s.SlashEpoch
}

func (m mstate) Equals(o market.DealState) bool {
return market.DealStatesEqual(m, o)
}

func (m MarketDealState) Iface() market.DealState {
return mstate{m}
}

type InvocResult struct {
Expand Down
File renamed without changes.
5 changes: 2 additions & 3 deletions cmd/boost/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,8 @@ func main() {
offlineDealCmd,
providerCmd,
walletCmd,
//TODO: enable when DDO ends up in a network upgrade
//directDealAllocate,
//directDealGetAllocations,
directDealAllocate,
directDealGetAllocations,
},
}
app.Setup()
Expand Down
9 changes: 5 additions & 4 deletions cmd/boost/util/util.god → cmd/boost/util/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,21 @@ package util
import (
"context"
"fmt"
"strconv"
"strings"

"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin"
"github.com/filecoin-project/go-state-types/builtin/v9/datacap"
"github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
"github.com/filecoin-project/go-state-types/builtin/v13/datacap"
"github.com/filecoin-project/go-state-types/builtin/v13/verifreg"
"github.com/filecoin-project/lotus/api"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
datacap2 "github.com/filecoin-project/lotus/chain/actors/builtin/datacap"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
"strconv"
"strings"
)

func CreateAllocationMsg(ctx context.Context, api api.Gateway, pInfos, miners []string, wallet address.Address, tmin, tmax, exp abi.ChainEpoch) (*types.Message, error) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ import (
"github.com/filecoin-project/boost/storagemarket/types"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin"
"github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/mitchellh/go-homedir"
Expand Down Expand Up @@ -48,20 +50,16 @@ var importDirectDataCmd = &cli.Command{
},
&cli.IntFlag{
Name: "start-epoch",
Usage: "start epoch by when the deal should be proved by provider on-chain",
Value: 35000, // default is 35000, handy for tests with 2k/devnet build
},
&cli.IntFlag{
Name: "duration",
Usage: "duration of the deal in epochs",
Value: 518400, // default is 2880 * 180 == 180 days
Usage: "start epoch by when the deal should be proved by provider on-chain (default: 2 days from now)",
},
},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() < 2 {
return fmt.Errorf("must specify piececid and file path")
}

ctx := cctx.Context

piececidStr := cctx.Args().Get(0)
path := cctx.Args().Get(1)

Expand Down Expand Up @@ -91,6 +89,17 @@ var importDirectDataCmd = &cli.Command{
}
defer closer()

lapi, lcloser, err := lcli.GetFullNodeAPIV1(cctx)
if err != nil {
return err
}
defer lcloser()

head, err := lapi.ChainHead(ctx)
if err != nil {
return fmt.Errorf("getting chain head: %w", err)
}

clientAddr, err := address.NewFromString(cctx.String("client-addr"))
if err != nil {
return fmt.Errorf("failed to parse clientaddr param: %w", err)
Expand All @@ -99,7 +108,21 @@ var importDirectDataCmd = &cli.Command{
allocationId := cctx.Uint64("allocation-id")

startEpoch := abi.ChainEpoch(cctx.Int("start-epoch"))
endEpoch := startEpoch + abi.ChainEpoch(cctx.Int("duration"))
// Set Default if not specified by the user
if startEpoch == 0 {
startEpoch = head.Height() + (builtin.EpochsInDay * 2)
}
alloc, err := lapi.StateGetAllocation(ctx, clientAddr, verifreg.AllocationId(allocationId), head.Key())
if err != nil {
return fmt.Errorf("getting claim details from chain: %w", err)
}

if alloc.Expiration < startEpoch {
return fmt.Errorf("allocation will expire on %d before start epoch %d", alloc.Expiration, startEpoch)
}

// Since StartEpoch is more than Head+StartEpochSealingBuffer, we can set end epoch as start+TermMin
endEpoch := startEpoch + alloc.TermMin

ddParams := types.DirectDealParams{
DealUUID: uuid.New(),
Expand Down
2 changes: 1 addition & 1 deletion cmd/boostd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func main() {
dummydealCmd,
indexProvCmd,
importDataCmd,
//importDirectDataCmd,
importDirectDataCmd,
logCmd,
netCmd,
pieceDirCmd,
Expand Down
6 changes: 3 additions & 3 deletions cmd/lib/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,23 +151,23 @@ func (a *MultiMinerAccessor) GetMinerAddresses() []address.Address {
func (a *MultiMinerAccessor) GetReader(ctx context.Context, minerAddr address.Address, id abi.SectorNumber, offset abi.PaddedPieceSize, length abi.PaddedPieceSize) (types.SectionReader, error) {
pr, ok := a.readers[minerAddr]
if !ok {
return nil, fmt.Errorf("get reader: no endpoint registered for miner %s", minerAddr)
return nil, fmt.Errorf("get reader: no endpoint registered for miner %s, len(readers)=%d", minerAddr, len(a.readers))
}
return pr.GetReader(ctx, minerAddr, id, offset, length)
}

func (a *MultiMinerAccessor) UnsealSectorAt(ctx context.Context, minerAddr address.Address, sectorID abi.SectorNumber, pieceOffset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) {
sa, ok := a.sas[minerAddr]
if !ok {
return nil, fmt.Errorf("read sector: no endpoint registered for miner %s", minerAddr)
return nil, fmt.Errorf("read sector: no endpoint registered for miner %s, len(readers)=%d", minerAddr, len(a.readers))
}
return sa.UnsealSectorAt(ctx, sectorID, pieceOffset, length)
}

func (a *MultiMinerAccessor) IsUnsealed(ctx context.Context, minerAddr address.Address, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) {
sa, ok := a.sas[minerAddr]
if !ok {
return false, fmt.Errorf("is unsealed: no endpoint registered for miner %s", minerAddr)
return false, fmt.Errorf("is unsealed: no endpoint registered for miner %s, len(readers)=%d", minerAddr, len(a.readers))
}
return sa.IsUnsealed(ctx, sectorID, offset, length)
}
Expand Down
1 change: 1 addition & 0 deletions docker/devnet/lotus-miner/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ ENV SECTOR_SIZE=8388608

VOLUME /var/tmp/filecoin-proof-parameters
VOLUME /var/lib/genesis
VOLUME /var/lib/builtin-actors

WORKDIR /app
RUN mkdir -p /app
Expand Down
1 change: 1 addition & 0 deletions docker/devnet/lotus/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ ENV LOTUS_FEVM_ENABLEETHRPC=true

VOLUME /var/tmp/filecoin-proof-parameters
VOLUME /var/lib/genesis
VOLUME /var/lib/builtin-actors

WORKDIR /app
RUN mkdir -p /app
Expand Down
Loading
Loading