From d8da3536c71d083c3dc024541db3997c317f10e2 Mon Sep 17 00:00:00 2001 From: Mike <41407352+hunjixin@users.noreply.github.com> Date: Fri, 8 Jul 2022 16:43:19 +0800 Subject: [PATCH] feat:add new api DagstoreInitializeStorage to load shard from piece storage (#5032) * feat:add api DagstoreInitializeStorage * fix:change dag cmd api permission to admin --- venus-shared/api/market/api.go | 11 +++--- venus-shared/api/market/method.md | 38 +++++++++++++++++--- venus-shared/api/market/mock/mock_imarket.go | 15 ++++++++ venus-shared/api/market/proxy_gen.go | 12 ++++--- 4 files changed, 64 insertions(+), 12 deletions(-) diff --git a/venus-shared/api/market/api.go b/venus-shared/api/market/api.go index 614f31547d..5763e56d4c 100644 --- a/venus-shared/api/market/api.go +++ b/venus-shared/api/market/api.go @@ -88,7 +88,7 @@ type IMarket interface { // DagstoreListShards returns information about all shards known to the // DAG store. Only available on nodes running the markets subsystem. - DagstoreListShards(ctx context.Context) ([]market.DagstoreShardInfo, error) //perm:read + DagstoreListShards(ctx context.Context) ([]market.DagstoreShardInfo, error) //perm:admin // DagstoreInitializeShard initializes an uninitialized shard. // @@ -107,14 +107,14 @@ type IMarket interface { // // This operation fails if the shard is not in ShardStateNew state. // It blocks until initialization finishes. - DagstoreInitializeShard(ctx context.Context, key string) error //perm:write + DagstoreInitializeShard(ctx context.Context, key string) error //perm:admin // DagstoreRecoverShard attempts to recover a failed shard. // // This operation fails if the shard is not in ShardStateErrored state. // It blocks until recovery finishes. If recovery failed, it returns the // error. - DagstoreRecoverShard(ctx context.Context, key string) error //perm:write + DagstoreRecoverShard(ctx context.Context, key string) error //perm:admin // DagstoreInitializeAll initializes all uninitialized shards in bulk, // according to the policy passed in the parameters. @@ -123,7 +123,10 @@ type IMarket interface { // IO pressure if the storage subsystem has a large amount of deals. // // It returns a stream of events to report progress. - DagstoreInitializeAll(ctx context.Context, params market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) //perm:write + DagstoreInitializeAll(ctx context.Context, params market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) //perm:admin + + //DagstoreInitializeStorage initializes all pieces in specify storage + DagstoreInitializeStorage(context.Context, string, market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) //perm:admin // DagstoreGC runs garbage collection on the DAG store. DagstoreGC(ctx context.Context) ([]market.DagstoreShardResult, error) //perm:admin diff --git a/venus-shared/api/market/method.md b/venus-shared/api/market/method.md index 35aa2d4e85..e18580fac1 100644 --- a/venus-shared/api/market/method.md +++ b/venus-shared/api/market/method.md @@ -10,6 +10,7 @@ * [DagstoreGC](#DagstoreGC) * [DagstoreInitializeAll](#DagstoreInitializeAll) * [DagstoreInitializeShard](#DagstoreInitializeShard) + * [DagstoreInitializeStorage](#DagstoreInitializeStorage) * [DagstoreListShards](#DagstoreListShards) * [DagstoreRecoverShard](#DagstoreRecoverShard) * [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals) @@ -242,7 +243,7 @@ IO pressure if the storage subsystem has a large amount of deals. It returns a stream of events to report progress. -Perms: write +Perms: admin Inputs: ```json @@ -286,7 +287,7 @@ This operation fails if the shard is not in ShardStateNew state. It blocks until initialization finishes. -Perms: write +Perms: admin Inputs: ```json @@ -297,12 +298,41 @@ Inputs: Response: `{}` +### DagstoreInitializeStorage +DagstoreInitializeStorage initializes all pieces in specify storage + + +Perms: admin + +Inputs: +```json +[ + "string value", + { + "MaxConcurrency": 123, + "IncludeSealed": true + } +] +``` + +Response: +```json +{ + "Key": "string value", + "Event": "string value", + "Success": true, + "Error": "string value", + "Total": 123, + "Current": 123 +} +``` + ### DagstoreListShards DagstoreListShards returns information about all shards known to the DAG store. Only available on nodes running the markets subsystem. -Perms: read +Perms: admin Inputs: `[]` @@ -325,7 +355,7 @@ It blocks until recovery finishes. If recovery failed, it returns the error. -Perms: write +Perms: admin Inputs: ```json diff --git a/venus-shared/api/market/mock/mock_imarket.go b/venus-shared/api/market/mock/mock_imarket.go index b8167d1ba2..9a6109b97d 100644 --- a/venus-shared/api/market/mock/mock_imarket.go +++ b/venus-shared/api/market/mock/mock_imarket.go @@ -181,6 +181,21 @@ func (mr *MockIMarketMockRecorder) DagstoreInitializeShard(arg0, arg1 interface{ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DagstoreInitializeShard", reflect.TypeOf((*MockIMarket)(nil).DagstoreInitializeShard), arg0, arg1) } +// DagstoreInitializeStorage mocks base method. +func (m *MockIMarket) DagstoreInitializeStorage(arg0 context.Context, arg1 string, arg2 market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DagstoreInitializeStorage", arg0, arg1, arg2) + ret0, _ := ret[0].(<-chan market.DagstoreInitializeAllEvent) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DagstoreInitializeStorage indicates an expected call of DagstoreInitializeStorage. +func (mr *MockIMarketMockRecorder) DagstoreInitializeStorage(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DagstoreInitializeStorage", reflect.TypeOf((*MockIMarket)(nil).DagstoreInitializeStorage), arg0, arg1, arg2) +} + // DagstoreListShards mocks base method. func (m *MockIMarket) DagstoreListShards(arg0 context.Context) ([]market.DagstoreShardInfo, error) { m.ctrl.T.Helper() diff --git a/venus-shared/api/market/proxy_gen.go b/venus-shared/api/market/proxy_gen.go index 8afd0bb349..46ab60626b 100644 --- a/venus-shared/api/market/proxy_gen.go +++ b/venus-shared/api/market/proxy_gen.go @@ -29,10 +29,11 @@ type IMarketStruct struct { AddS3PieceStorage func(ctx context.Context, readonly bool, endpoit, name, key, secret, token string) error `perm:"admin"` AssignUnPackedDeals func(ctx context.Context, sid abi.SectorID, ssize abi.SectorSize, spec *market.GetDealSpec) ([]*market.DealInfoIncludePath, error) `perm:"write"` DagstoreGC func(ctx context.Context) ([]market.DagstoreShardResult, error) `perm:"admin"` - DagstoreInitializeAll func(ctx context.Context, params market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) `perm:"write"` - DagstoreInitializeShard func(ctx context.Context, key string) error `perm:"write"` - DagstoreListShards func(ctx context.Context) ([]market.DagstoreShardInfo, error) `perm:"read"` - DagstoreRecoverShard func(ctx context.Context, key string) error `perm:"write"` + DagstoreInitializeAll func(ctx context.Context, params market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) `perm:"admin"` + DagstoreInitializeShard func(ctx context.Context, key string) error `perm:"admin"` + DagstoreInitializeStorage func(context.Context, string, market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) `perm:"admin"` + DagstoreListShards func(ctx context.Context) ([]market.DagstoreShardInfo, error) `perm:"admin"` + DagstoreRecoverShard func(ctx context.Context, key string) error `perm:"admin"` DealsConsiderOfflineRetrievalDeals func(context.Context) (bool, error) `perm:"admin"` DealsConsiderOfflineStorageDeals func(context.Context) (bool, error) `perm:"admin"` DealsConsiderOnlineRetrievalDeals func(context.Context) (bool, error) `perm:"admin"` @@ -126,6 +127,9 @@ func (s *IMarketStruct) DagstoreInitializeAll(p0 context.Context, p1 market.Dags func (s *IMarketStruct) DagstoreInitializeShard(p0 context.Context, p1 string) error { return s.Internal.DagstoreInitializeShard(p0, p1) } +func (s *IMarketStruct) DagstoreInitializeStorage(p0 context.Context, p1 string, p2 market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) { + return s.Internal.DagstoreInitializeStorage(p0, p1, p2) +} func (s *IMarketStruct) DagstoreListShards(p0 context.Context) ([]market.DagstoreShardInfo, error) { return s.Internal.DagstoreListShards(p0) }