From 3c4213016214ec250d0d55c9487c059e6af3060b Mon Sep 17 00:00:00 2001 From: hunjixin <1084400399@qq.com> Date: Tue, 5 Jul 2022 15:40:44 +0800 Subject: [PATCH 1/2] feat:add api DagstoreInitializeStorage --- venus-shared/api/market/api.go | 3 ++ venus-shared/api/market/method.md | 29 ++++++++++++++++++++ venus-shared/api/market/mock/mock_imarket.go | 15 ++++++++++ venus-shared/api/market/proxy_gen.go | 4 +++ 4 files changed, 51 insertions(+) diff --git a/venus-shared/api/market/api.go b/venus-shared/api/market/api.go index 614f31547d..fb95338d66 100644 --- a/venus-shared/api/market/api.go +++ b/venus-shared/api/market/api.go @@ -125,6 +125,9 @@ type IMarket interface { // It returns a stream of events to report progress. DagstoreInitializeAll(ctx context.Context, params market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) //perm:write + //DagstoreInitializeStorage initializes all pieces in specify storage + DagstoreInitializeStorage(context.Context, string, market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) + // DagstoreGC runs garbage collection on the DAG store. DagstoreGC(ctx context.Context) ([]market.DagstoreShardResult, error) //perm:admin diff --git a/venus-shared/api/market/method.md b/venus-shared/api/market/method.md index 35aa2d4e85..3ff5287c11 100644 --- a/venus-shared/api/market/method.md +++ b/venus-shared/api/market/method.md @@ -10,6 +10,7 @@ * [DagstoreGC](#DagstoreGC) * [DagstoreInitializeAll](#DagstoreInitializeAll) * [DagstoreInitializeShard](#DagstoreInitializeShard) + * [DagstoreInitializeStorage](#DagstoreInitializeStorage) * [DagstoreListShards](#DagstoreListShards) * [DagstoreRecoverShard](#DagstoreRecoverShard) * [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals) @@ -297,6 +298,34 @@ Inputs: Response: `{}` +### DagstoreInitializeStorage + + +Perms: + +Inputs: +```json +[ + "string value", + { + "MaxConcurrency": 123, + "IncludeSealed": true + } +] +``` + +Response: +```json +{ + "Key": "string value", + "Event": "string value", + "Success": true, + "Error": "string value", + "Total": 123, + "Current": 123 +} +``` + ### DagstoreListShards DagstoreListShards returns information about all shards known to the DAG store. Only available on nodes running the markets subsystem. diff --git a/venus-shared/api/market/mock/mock_imarket.go b/venus-shared/api/market/mock/mock_imarket.go index b8167d1ba2..9a6109b97d 100644 --- a/venus-shared/api/market/mock/mock_imarket.go +++ b/venus-shared/api/market/mock/mock_imarket.go @@ -181,6 +181,21 @@ func (mr *MockIMarketMockRecorder) DagstoreInitializeShard(arg0, arg1 interface{ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DagstoreInitializeShard", reflect.TypeOf((*MockIMarket)(nil).DagstoreInitializeShard), arg0, arg1) } +// DagstoreInitializeStorage mocks base method. +func (m *MockIMarket) DagstoreInitializeStorage(arg0 context.Context, arg1 string, arg2 market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DagstoreInitializeStorage", arg0, arg1, arg2) + ret0, _ := ret[0].(<-chan market.DagstoreInitializeAllEvent) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DagstoreInitializeStorage indicates an expected call of DagstoreInitializeStorage. +func (mr *MockIMarketMockRecorder) DagstoreInitializeStorage(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DagstoreInitializeStorage", reflect.TypeOf((*MockIMarket)(nil).DagstoreInitializeStorage), arg0, arg1, arg2) +} + // DagstoreListShards mocks base method. func (m *MockIMarket) DagstoreListShards(arg0 context.Context) ([]market.DagstoreShardInfo, error) { m.ctrl.T.Helper() diff --git a/venus-shared/api/market/proxy_gen.go b/venus-shared/api/market/proxy_gen.go index 8afd0bb349..952ab5fb8c 100644 --- a/venus-shared/api/market/proxy_gen.go +++ b/venus-shared/api/market/proxy_gen.go @@ -31,6 +31,7 @@ type IMarketStruct struct { DagstoreGC func(ctx context.Context) ([]market.DagstoreShardResult, error) `perm:"admin"` DagstoreInitializeAll func(ctx context.Context, params market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) `perm:"write"` DagstoreInitializeShard func(ctx context.Context, key string) error `perm:"write"` + DagstoreInitializeStorage func(context.Context, string, market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) `perm:""` DagstoreListShards func(ctx context.Context) ([]market.DagstoreShardInfo, error) `perm:"read"` DagstoreRecoverShard func(ctx context.Context, key string) error `perm:"write"` DealsConsiderOfflineRetrievalDeals func(context.Context) (bool, error) `perm:"admin"` @@ -126,6 +127,9 @@ func (s *IMarketStruct) DagstoreInitializeAll(p0 context.Context, p1 market.Dags func (s *IMarketStruct) DagstoreInitializeShard(p0 context.Context, p1 string) error { return s.Internal.DagstoreInitializeShard(p0, p1) } +func (s *IMarketStruct) DagstoreInitializeStorage(p0 context.Context, p1 string, p2 market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) { + return s.Internal.DagstoreInitializeStorage(p0, p1, p2) +} func (s *IMarketStruct) DagstoreListShards(p0 context.Context) ([]market.DagstoreShardInfo, error) { return s.Internal.DagstoreListShards(p0) } From 7e4508e027cd38fc65686a84a2aba6c133430434 Mon Sep 17 00:00:00 2001 From: hunjixin <1084400399@qq.com> Date: Wed, 6 Jul 2022 15:48:14 +0800 Subject: [PATCH 2/2] fix:change dag cmd api permission to admin --- venus-shared/api/market/api.go | 10 +++++----- venus-shared/api/market/method.md | 11 ++++++----- venus-shared/api/market/proxy_gen.go | 10 +++++----- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/venus-shared/api/market/api.go b/venus-shared/api/market/api.go index fb95338d66..5763e56d4c 100644 --- a/venus-shared/api/market/api.go +++ b/venus-shared/api/market/api.go @@ -88,7 +88,7 @@ type IMarket interface { // DagstoreListShards returns information about all shards known to the // DAG store. Only available on nodes running the markets subsystem. - DagstoreListShards(ctx context.Context) ([]market.DagstoreShardInfo, error) //perm:read + DagstoreListShards(ctx context.Context) ([]market.DagstoreShardInfo, error) //perm:admin // DagstoreInitializeShard initializes an uninitialized shard. // @@ -107,14 +107,14 @@ type IMarket interface { // // This operation fails if the shard is not in ShardStateNew state. // It blocks until initialization finishes. - DagstoreInitializeShard(ctx context.Context, key string) error //perm:write + DagstoreInitializeShard(ctx context.Context, key string) error //perm:admin // DagstoreRecoverShard attempts to recover a failed shard. // // This operation fails if the shard is not in ShardStateErrored state. // It blocks until recovery finishes. If recovery failed, it returns the // error. - DagstoreRecoverShard(ctx context.Context, key string) error //perm:write + DagstoreRecoverShard(ctx context.Context, key string) error //perm:admin // DagstoreInitializeAll initializes all uninitialized shards in bulk, // according to the policy passed in the parameters. @@ -123,10 +123,10 @@ type IMarket interface { // IO pressure if the storage subsystem has a large amount of deals. // // It returns a stream of events to report progress. - DagstoreInitializeAll(ctx context.Context, params market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) //perm:write + DagstoreInitializeAll(ctx context.Context, params market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) //perm:admin //DagstoreInitializeStorage initializes all pieces in specify storage - DagstoreInitializeStorage(context.Context, string, market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) + DagstoreInitializeStorage(context.Context, string, market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) //perm:admin // DagstoreGC runs garbage collection on the DAG store. DagstoreGC(ctx context.Context) ([]market.DagstoreShardResult, error) //perm:admin diff --git a/venus-shared/api/market/method.md b/venus-shared/api/market/method.md index 3ff5287c11..e18580fac1 100644 --- a/venus-shared/api/market/method.md +++ b/venus-shared/api/market/method.md @@ -243,7 +243,7 @@ IO pressure if the storage subsystem has a large amount of deals. It returns a stream of events to report progress. -Perms: write +Perms: admin Inputs: ```json @@ -287,7 +287,7 @@ This operation fails if the shard is not in ShardStateNew state. It blocks until initialization finishes. -Perms: write +Perms: admin Inputs: ```json @@ -299,9 +299,10 @@ Inputs: Response: `{}` ### DagstoreInitializeStorage +DagstoreInitializeStorage initializes all pieces in specify storage -Perms: +Perms: admin Inputs: ```json @@ -331,7 +332,7 @@ DagstoreListShards returns information about all shards known to the DAG store. Only available on nodes running the markets subsystem. -Perms: read +Perms: admin Inputs: `[]` @@ -354,7 +355,7 @@ It blocks until recovery finishes. If recovery failed, it returns the error. -Perms: write +Perms: admin Inputs: ```json diff --git a/venus-shared/api/market/proxy_gen.go b/venus-shared/api/market/proxy_gen.go index 952ab5fb8c..46ab60626b 100644 --- a/venus-shared/api/market/proxy_gen.go +++ b/venus-shared/api/market/proxy_gen.go @@ -29,11 +29,11 @@ type IMarketStruct struct { AddS3PieceStorage func(ctx context.Context, readonly bool, endpoit, name, key, secret, token string) error `perm:"admin"` AssignUnPackedDeals func(ctx context.Context, sid abi.SectorID, ssize abi.SectorSize, spec *market.GetDealSpec) ([]*market.DealInfoIncludePath, error) `perm:"write"` DagstoreGC func(ctx context.Context) ([]market.DagstoreShardResult, error) `perm:"admin"` - DagstoreInitializeAll func(ctx context.Context, params market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) `perm:"write"` - DagstoreInitializeShard func(ctx context.Context, key string) error `perm:"write"` - DagstoreInitializeStorage func(context.Context, string, market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) `perm:""` - DagstoreListShards func(ctx context.Context) ([]market.DagstoreShardInfo, error) `perm:"read"` - DagstoreRecoverShard func(ctx context.Context, key string) error `perm:"write"` + DagstoreInitializeAll func(ctx context.Context, params market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) `perm:"admin"` + DagstoreInitializeShard func(ctx context.Context, key string) error `perm:"admin"` + DagstoreInitializeStorage func(context.Context, string, market.DagstoreInitializeAllParams) (<-chan market.DagstoreInitializeAllEvent, error) `perm:"admin"` + DagstoreListShards func(ctx context.Context) ([]market.DagstoreShardInfo, error) `perm:"admin"` + DagstoreRecoverShard func(ctx context.Context, key string) error `perm:"admin"` DealsConsiderOfflineRetrievalDeals func(context.Context) (bool, error) `perm:"admin"` DealsConsiderOfflineStorageDeals func(context.Context) (bool, error) `perm:"admin"` DealsConsiderOnlineRetrievalDeals func(context.Context) (bool, error) `perm:"admin"`