diff --git a/.github/workflows/layotto-ci.yml b/.github/workflows/layotto-ci.yml index e1dc7bd5e4..d6efeedacb 100644 --- a/.github/workflows/layotto-ci.yml +++ b/.github/workflows/layotto-ci.yml @@ -31,19 +31,34 @@ jobs: - name: Format Go run: | make check.style - + resolve-modules: + name: "🌈️ Resolve Go Modules" + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout Sources + uses: actions/checkout@v2 + - id: set-matrix + run: sh ./etc/script/resolve-modules.sh golangci-lint: name: "👀 Go CI Linter" - needs: [style-check] + needs: [style-check,resolve-modules] runs-on: ubuntu-20.04 + strategy: + matrix: ${{ fromJson(needs.resolve-modules.outputs.matrix) }} steps: - uses: actions/checkout@v3 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.18.1 + go-version: 1.14.15 - name: Go Lint Test - run: make check.lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.46.2 + working-directory: ${{ matrix.workdir }} + args: "--out-${NO_FUTURE}format colored-line-number" go-unit-test: name: "👀 Go Unit Test" @@ -66,7 +81,7 @@ jobs: - name: Check out code uses: actions/checkout@v2 - name: Test Coverage - run: make style.coverage + run: make style.coverage - name: Post Coverage run: bash <(curl -s https://codecov.io/bash) @@ -247,7 +262,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -271,7 +286,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/layotto-release.yml b/.github/workflows/layotto-release.yml index 0af89cc9b7..7e4a3eff82 100644 --- a/.github/workflows/layotto-release.yml +++ b/.github/workflows/layotto-release.yml @@ -18,20 +18,35 @@ jobs: - name: Format Go run: | make check.style - + + resolve-modules: + name: "👀 Resolve Go Modules" + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout Sources + uses: actions/checkout@v2 + - id: set-matrix + run: sh ./etc/script/resolve-modules.sh golangci-lint: name: "👀 Go CI Linter" - needs: [style-check] + needs: [ style-check,resolve-modules ] runs-on: ubuntu-20.04 + strategy: + matrix: ${{ fromJson(needs.resolve-modules.outputs.matrix) }} steps: - - uses: actions/checkout@v3 - - name: Set up Go - if: ${{ startsWith(github.ref, 'refs/tags/') }} - uses: actions/setup-go@v3 - with: - go-version: 1.18.1 - - name: Go Lint Test - run: make check.lint + - uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.14.15 + - name: Go Lint Test + uses: golangci/golangci-lint-action@v3 + with: + version: v1.46.2 + working-directory: ${{ matrix.workdir }} + args: "--out-${NO_FUTURE}format colored-line-number" go-unit-test: name: "👀 Go Unit Test" diff --git a/.golangci.yml b/.golangci.yml index e6c7e98384..394e9f8c61 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,6 +7,7 @@ run : - pkg/mock skip-files : - .pb.go + - rpc/invoker/mosn/transport_protocol/bolt.go linters : diff --git a/components/configstores/apollo/configstore.go b/components/configstores/apollo/configstore.go index 33c52c7562..2c1f3db649 100644 --- a/components/configstores/apollo/configstore.go +++ b/components/configstores/apollo/configstore.go @@ -41,7 +41,10 @@ var ( livenessIndicator *actuators.HealthIndicator ) -const componentName = "apollo" +const ( + defaultGroup = "application" + componentName = "apollo" +) func init() { readinessIndicator = actuators.NewHealthIndicator() @@ -58,8 +61,8 @@ type ConfigStore struct { listener *changeListener kvRepo Repository tagsRepo Repository - kvConfig *RepoConfig - tagsConfig *RepoConfig + kvConfig *repoConfig + tagsConfig *repoConfig openAPIClient httpClient } type httpClient interface { @@ -75,7 +78,7 @@ func (c *httpClientImpl) Do(req *http.Request) (*http.Response, error) { } func (c *ConfigStore) GetDefaultGroup() string { - return "application" + return defaultGroup } func (c *ConfigStore) GetDefaultLabel() string { @@ -167,7 +170,7 @@ func (c *ConfigStore) doInit(config *configstores.StoreConfig) error { } // TODO make 'env' configurable // 2. SetConfig client - kvRepoConfig := &RepoConfig{ + kvRepoConfig := &repoConfig{ addr: addr, appId: appId, env: c.env, @@ -472,6 +475,9 @@ func (c *ConfigStore) setItem(appId string, item *configstores.ConfigurationItem return err } req, err := http.NewRequest("PUT", setUrl, strings.NewReader(string(reqBodyJson))) + if err != nil { + return err + } // add params q := req.URL.Query() q.Add("createIfNotExists", "true") @@ -535,6 +541,9 @@ func (c *ConfigStore) commit(env string, appId string, cluster string, namespace return err } req, err := http.NewRequest("POST", commitUrl, strings.NewReader(string(reqBodyJson))) + if err != nil { + return err + } // add headers c.addHeaderForOpenAPI(req) // do request @@ -553,6 +562,9 @@ func (c *ConfigStore) deleteItem(env string, appId string, cluster string, group keyWithLabel := c.concatenateKey(key, label) deleteUrl := fmt.Sprintf(deleteUrlTpl, c.openAPIAddress, env, appId, cluster, group, keyWithLabel) req, err := http.NewRequest("DELETE", deleteUrl, nil) + if err != nil { + return err + } // add params q := req.URL.Query() q.Add("key", keyWithLabel) @@ -571,7 +583,7 @@ func (c *ConfigStore) deleteItem(env string, appId string, cluster string, group return err } -func (c *ConfigStore) initTagsClient(tagCfg *RepoConfig) error { +func (c *ConfigStore) initTagsClient(tagCfg *repoConfig) error { // 1. create if not exist err := c.createNamespace(c.env, tagCfg.appId, tagCfg.cluster, c.tagsNamespace) if err != nil { @@ -599,6 +611,9 @@ func (c *ConfigStore) createNamespace(env string, appId string, cluster string, return err } req, err := http.NewRequest("POST", url, strings.NewReader(string(reqBodyJson))) + if err != nil { + return err + } // add headers c.addHeaderForOpenAPI(req) log.DefaultLogger.Debugf("createNamespace url: %v, request body: %s, request: %+v", url, reqBodyJson, req) diff --git a/components/configstores/apollo/configstore_test.go b/components/configstores/apollo/configstore_test.go index b095106724..4615676570 100644 --- a/components/configstores/apollo/configstore_test.go +++ b/components/configstores/apollo/configstore_test.go @@ -27,16 +27,18 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/zouyx/agollo/v4" "mosn.io/mosn/pkg/log" "mosn.io/layotto/components/configstores" ) +const ( + prod = "prod" +) + // MockRepository implements Repository interface type MockRepository struct { - client *agollo.Client - cfg *RepoConfig + cfg *repoConfig invoked []string cache map[string]map[string]string } @@ -46,14 +48,10 @@ func (a *MockRepository) Connect() error { return err } -func (a *MockRepository) SetConfig(r *RepoConfig) { +func (a *MockRepository) SetConfig(r *repoConfig) { a.cfg = r } -func (a *MockRepository) GetConfig() *RepoConfig { - return a.cfg -} - func newMockRepository() *MockRepository { return &MockRepository{ invoked: make([]string, 0, 2), @@ -70,7 +68,7 @@ func (a *MockRepository) Get(namespace string, key string) (interface{}, error) if _, ok := a.cache[namespace]; !ok { a.cache[namespace] = make(map[string]string) } - v, _ := a.cache[namespace][key] + v := a.cache[namespace][key] return v, nil } @@ -130,8 +128,8 @@ func TestConfigStore_read(t *testing.T) { // get key var req configstores.GetRequest req.AppId = appId - req.Group = "application" - req.Label = "prod" + req.Group = defaultGroup + req.Label = prod req.Keys = []string{"sofa"} resp, err := store.Get(context.Background(), &req) if err != nil || len(resp) == 0 || resp[0].Content != "sofa@$prod" { @@ -171,14 +169,14 @@ func TestConfigStore_read(t *testing.T) { var subReq configstores.SubscribeReq ch := make(chan *configstores.SubscribeResp) subReq.AppId = "testApplication_yang" - subReq.Group = "application" - subReq.Label = "prod" + subReq.Group = defaultGroup + subReq.Label = prod subReq.Keys = []string{"sofa"} err = store.Subscribe(&subReq, ch) if err != nil { t.Error(err) } - subReq.Group = "application" + subReq.Group = defaultGroup subReq.Label = "" subReq.Keys = []string{} err = store.Subscribe(&subReq, ch) @@ -209,6 +207,24 @@ func TestConfigStore_Init(t *testing.T) { err := store.Init(cfg) assert.NotNil(t, err) }) + t.Run("when open_api_token blank then error", func(t *testing.T) { + // 1. set up + // inject the MockRepository into a ConfigStore + store, cfg := setup(t) + cfg.Metadata["open_api_token"] = "" + store.openAPIClient = newMockHttpClient(http.StatusBadRequest) + kvRepo := store.kvRepo.(*MockRepository) + kvRepo.Set("application", "sofa@$prod", "sofa@$prod") + kvRepo.Set("application", "apollo@$prod", "apollo@$prod") + kvRepo.Set("dubbo", "dubbo", "dubbo") + + // 2. test the ConfigStore,which has a MockRepository in it + // init + log.DefaultLogger.SetLogLevel(log.DEBUG) + err := store.Init(cfg) + assert.Error(t, err) + }) + t.Run("when namespace exist then succeed with debug information", func(t *testing.T) { // 1. set up // inject the MockRepository into a ConfigStore @@ -299,8 +315,8 @@ func TestConfigStore_write(t *testing.T) { req.AppId = appId item.Key = "sofa" item.Content = "v1" - item.Group = "application" - item.Label = "prod" + item.Group = defaultGroup + item.Label = prod req.StoreName = "apollo" req.Items = append(req.Items, &item) @@ -352,8 +368,8 @@ func TestConfigStore_write(t *testing.T) { var delReq configstores.DeleteRequest delReq.AppId = appId delReq.Keys = []string{"sofa"} - delReq.Group = "application" - delReq.Label = "prod" + delReq.Group = defaultGroup + delReq.Label = prod err = store.Delete(context.Background(), &delReq) if err != nil { t.Error(err) @@ -375,7 +391,7 @@ func TestConfigStore_Init_fail(t *testing.T) { err := store.Init(cfg) assert.Nil(t, err) // no config - store, cfg = setup(t) + store, _ = setup(t) err = store.Init(nil) if notNil := assert.NotNil(t, err); notNil { assert.True(t, err.Error() != "") diff --git a/components/configstores/apollo/repository.go b/components/configstores/apollo/repository.go index be1bd83185..67997f9a01 100644 --- a/components/configstores/apollo/repository.go +++ b/components/configstores/apollo/repository.go @@ -26,8 +26,7 @@ import ( // An interface to abstract different apollo sdks,also making it easier to write unit tests. type Repository interface { - SetConfig(r *RepoConfig) - GetConfig() *RepoConfig + SetConfig(r *repoConfig) Connect() error // subscribe AddChangeListener(listener *changeListener) @@ -37,15 +36,15 @@ type Repository interface { Range(namespace string, f func(key, value interface{}) bool) error } -type RepoConfig struct { - addr string `json:"addr"` - appId string `json:"appId"` - env string `json:"env"` - cluster string `json:"cluster"` - namespaceName string `json:"namespaceName"` +type repoConfig struct { + addr string + appId string + env string + cluster string + namespaceName string // whether backup config after fetch config from apollo - isBackupConfig bool `default:"true" json:"isBackupConfig"` - secret string `json:"secret"` + isBackupConfig bool + secret string } func init() { @@ -55,7 +54,7 @@ func init() { //Implement Repository interface type AgolloRepository struct { client *agollo.Client - cfg *RepoConfig + cfg *repoConfig } func (a *AgolloRepository) Connect() error { @@ -66,11 +65,11 @@ func (a *AgolloRepository) Connect() error { return err } -func (a *AgolloRepository) SetConfig(r *RepoConfig) { +func (a *AgolloRepository) SetConfig(r *repoConfig) { a.cfg = r } -func repoConfig2AgolloConfig(r *RepoConfig) *agolloConfig.AppConfig { +func repoConfig2AgolloConfig(r *repoConfig) *agolloConfig.AppConfig { return &agolloConfig.AppConfig{ IP: r.addr, AppID: r.appId, @@ -81,10 +80,6 @@ func repoConfig2AgolloConfig(r *RepoConfig) *agolloConfig.AppConfig { } } -func (a *AgolloRepository) GetConfig() *RepoConfig { - return a.cfg -} - func newAgolloRepository() Repository { return &AgolloRepository{} } diff --git a/components/configstores/etcdv3/etcdv3.go b/components/configstores/etcdv3/etcdv3.go index e06e655029..834fc06e0d 100644 --- a/components/configstores/etcdv3/etcdv3.go +++ b/components/configstores/etcdv3/etcdv3.go @@ -31,8 +31,12 @@ import ( "mosn.io/layotto/components/trace" ) +const ( + defaultGroup = "default" + defaultLabel = "default" +) + type EtcdV3ConfigStore struct { - name string client *clientv3.Client sync.RWMutex subscribeKey map[string]string @@ -44,11 +48,11 @@ type EtcdV3ConfigStore struct { } func (c *EtcdV3ConfigStore) GetDefaultGroup() string { - return "default" + return defaultGroup } func (c *EtcdV3ConfigStore) GetDefaultLabel() string { - return "default" + return defaultLabel } func NewStore() configstores.Store { diff --git a/components/configstores/etcdv3/etcdv3_test.go b/components/configstores/etcdv3/etcdv3_test.go index e274fba38b..aad51e213e 100644 --- a/components/configstores/etcdv3/etcdv3_test.go +++ b/components/configstores/etcdv3/etcdv3_test.go @@ -30,7 +30,10 @@ import ( "mosn.io/layotto/components/configstores" ) -const defaultEtcdV3WorkDir = "/tmp/default-dubbo-go-remote.etcd" +const ( + appId = "mosn" + defaultEtcdV3WorkDir = "/tmp/default-dubbo-go-remote.etcd" +) var etcd EtcdV3ConfigStore @@ -53,7 +56,7 @@ func TestGetDetailInfoFromResult(t *testing.T) { kvs = append(kvs, kv3) kvs = append(kvs, kv4) kvs = append(kvs, kv5) - targetStr := []string{"mosn", "group1", "label1", "sofa"} + targetStr := []string{appId, "group1", "label1", "sofa"} res := etcd.GetItemsFromAllKeys(kvs, targetStr) for _, value := range res { assert.Equal(t, value.Group, "group1") @@ -62,7 +65,7 @@ func TestGetDetailInfoFromResult(t *testing.T) { assert.Equal(t, value.Content, "value1") assert.Equal(t, value.Tags, map[string]string{"tag1": "tag1", "tag2": "tag2"}) } - targetStr2 := []string{"mosn", "*", "label1", "sofa"} + targetStr2 := []string{appId, "*", "label1", "sofa"} res = etcd.GetItemsFromAllKeys(kvs, targetStr2) for _, value := range res { assert.Equal(t, value.Group, "group1") @@ -72,7 +75,7 @@ func TestGetDetailInfoFromResult(t *testing.T) { assert.Equal(t, value.Tags, map[string]string{"tag1": "tag1", "tag2": "tag2"}) } - targetStr3 := []string{"mosn", "*", "*", "sofa"} + targetStr3 := []string{appId, "*", "*", "sofa"} res = etcd.GetItemsFromAllKeys(kvs, targetStr3) for _, value := range res { assert.Equal(t, value.Group, "group1") @@ -156,31 +159,31 @@ func (suite *ClientTestSuite) SetupTest() { func (suite *ClientTestSuite) Delete() { var delReq configstores.DeleteRequest - delReq.AppId = "mosn" + delReq.AppId = appId delReq.Keys = []string{"sofa"} - delReq.Group = "default" - delReq.Label = "default" + delReq.Group = defaultGroup + delReq.Label = defaultLabel err := suite.store.Delete(context.Background(), &delReq) if err != nil { suite.T().Fatal(err) } var req configstores.GetRequest - req.AppId = "mosn" + req.AppId = appId req.Keys = []string{"sofa"} resp, err := suite.store.Get(context.Background(), &req) assert.Equal(suite.T(), len(resp), 0) - + assert.Nil(suite.T(), err) } func (suite *ClientTestSuite) Subscribe() { var subReq configstores.SubscribeReq var i int ch := make(chan *configstores.SubscribeResp) - subReq.AppId = "mosn" - subReq.Group = "default" - subReq.Label = "default" + subReq.AppId = appId + subReq.Group = defaultGroup + subReq.Label = defaultLabel subReq.Keys = []string{"sofa"} wg.Add(1) suite.store.Subscribe(&subReq, ch) @@ -200,9 +203,9 @@ func (suite *ClientTestSuite) Subscribe() { func (suite *ClientTestSuite) Get() { var req configstores.GetRequest - req.AppId = "mosn" - req.Group = "default" - req.Label = "default" + req.AppId = appId + req.Group = defaultGroup + req.Label = defaultLabel req.Keys = []string{"sofa"} resp, err := suite.store.Get(context.Background(), &req) if err != nil || len(resp) == 0 { @@ -211,8 +214,8 @@ func (suite *ClientTestSuite) Get() { for _, value := range resp { assert.Equal(suite.T(), value.Key, "sofa") assert.Equal(suite.T(), value.Content, "v1") - assert.Equal(suite.T(), value.Group, "default") - assert.Equal(suite.T(), value.Label, "default") + assert.Equal(suite.T(), value.Group, defaultGroup) + assert.Equal(suite.T(), value.Label, defaultLabel) } req.Keys = []string{} @@ -223,8 +226,8 @@ func (suite *ClientTestSuite) Get() { for _, value := range resp { assert.Equal(suite.T(), value.Key, "sofa") assert.Equal(suite.T(), value.Content, "v1") - assert.Equal(suite.T(), value.Group, "default") - assert.Equal(suite.T(), value.Label, "default") + assert.Equal(suite.T(), value.Group, defaultGroup) + assert.Equal(suite.T(), value.Label, defaultLabel) } } @@ -233,10 +236,10 @@ func (suite *ClientTestSuite) Set() { var item configstores.ConfigurationItem item.Key = "sofa" item.Content = "v1" - item.Group = "default" - item.Label = "default" + item.Group = defaultGroup + item.Label = defaultLabel req.StoreName = "etcd" - req.AppId = "mosn" + req.AppId = appId req.Items = append(req.Items, &item) err := suite.store.Set(context.Background(), &req) if err != nil { diff --git a/components/file/hdfs/hdfs.go b/components/file/hdfs/hdfs.go index 5244ef2773..1620b407b1 100644 --- a/components/file/hdfs/hdfs.go +++ b/components/file/hdfs/hdfs.go @@ -145,7 +145,7 @@ func (h *hdfs) List(ctx context.Context, request *file.ListRequest) (*file.ListR it, err := client.List(starter) if err != nil { - err = ErrHdfsListFail + return nil, ErrHdfsListFail } marker := "" diff --git a/components/file/hdfs/hdfs_test.go b/components/file/hdfs/hdfs_test.go index cc166e294a..65f513d460 100644 --- a/components/file/hdfs/hdfs_test.go +++ b/components/file/hdfs/hdfs_test.go @@ -203,11 +203,11 @@ func TestHdfs_List(t *testing.T) { assert.Nil(t, resp) req.Metadata["endpoint"] = endpoint - resp, err = hdfs.List(context.TODO(), req) + _, err = hdfs.List(context.TODO(), req) assert.Equal(t, ErrClientNotExist, err) req.Metadata["endpoint"] = tcpEndpoint - resp, err = hdfs.List(context.TODO(), req) + _, err = hdfs.List(context.TODO(), req) assert.NotNil(t, err) } @@ -232,11 +232,11 @@ func TestHdfs_Stat(t *testing.T) { assert.Nil(t, resp) req.Metadata["endpoint"] = endpoint - resp, err = hdfs.Stat(context.TODO(), req) + _, err = hdfs.Stat(context.TODO(), req) assert.Equal(t, ErrClientNotExist, err) req.Metadata["endpoint"] = tcpEndpoint - resp, err = hdfs.Stat(context.TODO(), req) + _, err = hdfs.Stat(context.TODO(), req) assert.NotNil(t, err) } diff --git a/components/file/local/file.go b/components/file/local/file.go index 230c3c5bd3..5cdd513631 100644 --- a/components/file/local/file.go +++ b/components/file/local/file.go @@ -67,7 +67,7 @@ func (lf *LocalStore) Put(ctx context.Context, f *file.PutFileStu) error { return err } defer fileObj.Close() - data := make([]byte, 512, 512) + data := make([]byte, 512) for { n, err := f.DataStream.Read(data) if err != nil { @@ -159,7 +159,7 @@ func (lf *LocalStore) Stat(ctx context.Context, f *file.FileMetaRequest) (*file. m := strconv.Itoa(mode) t := fileInfo.IsDir() isDir := "false" - if t == true { + if t { isDir = "true" } resp.Metadata[FileMode] = append(resp.Metadata[FileMode], m) diff --git a/components/file/local/file_test.go b/components/file/local/file_test.go index 4fdbd5244a..6a44a86a51 100644 --- a/components/file/local/file_test.go +++ b/components/file/local/file_test.go @@ -64,11 +64,12 @@ func TestFile(t *testing.T) { f.DataStream = reader go WriteFile(writer) err = ls.Put(context.TODO(), f) + assert.Nil(t, err) exist, err := CheckFileExist(f.FileName) assert.Nil(t, err) assert.Equal(t, true, exist) - data := make([]byte, 10, 10) + data := make([]byte, 10) fs := &file.GetFileStu{} fs.FileName = FileName stream, err := ls.Get(context.TODO(), fs) diff --git a/components/file/s3/alicloud/oss.go b/components/file/s3/alicloud/oss.go index fb345c0312..c2f538be44 100644 --- a/components/file/s3/alicloud/oss.go +++ b/components/file/s3/alicloud/oss.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "strconv" - "sync" "github.com/aliyun/aliyun-oss-go-sdk/oss" @@ -32,7 +31,6 @@ import ( const ( endpointKey = "endpoint" - bucketKey = "bucket" storageTypeKey = "storageType" ) @@ -40,7 +38,6 @@ const ( type AliCloudOSS struct { metadata map[string]*OssMetadata client map[string]*oss.Client - stream sync.Map } type OssMetadata struct { @@ -209,8 +206,7 @@ func (s *AliCloudOSS) getClient(metadata *OssMetadata) (*oss.Client, error) { } func (s *AliCloudOSS) getBucket(fileName string, metaData map[string]string) (*oss.Bucket, error) { - ossClient := &oss.Client{} - bucket := &oss.Bucket{} + var ossClient *oss.Client var err error // get oss client if _, ok := metaData[endpointKey]; ok { @@ -228,7 +224,7 @@ func (s *AliCloudOSS) getBucket(fileName string, metaData map[string]string) (*o if err != nil { return nil, err } - bucket, err = ossClient.Bucket(bucketName) + bucket, err := ossClient.Bucket(bucketName) if err != nil { return nil, err } diff --git a/components/file/s3/alicloud/oss_test.go b/components/file/s3/alicloud/oss_test.go index e662aa6aba..c7ad7c5da1 100644 --- a/components/file/s3/alicloud/oss_test.go +++ b/components/file/s3/alicloud/oss_test.go @@ -28,13 +28,16 @@ import ( "mosn.io/layotto/components/file" ) -const data = `[ +const ( + data = `[ { "endpoint": "endpoint_address", "accessKeyID": "accessKey", "accessKeySecret": "secret" } ]` + fileName = "b/a.txt" +) func TestInit(t *testing.T) { fc := file.FileConfig{} @@ -80,7 +83,7 @@ func TestGetBucket(t *testing.T) { assert.Equal(t, err.Error(), "invalid fileName format") assert.Nil(t, bucket) - bucket, err = ac.getBucket("b/a.txt", mt) + bucket, err = ac.getBucket(fileName, mt) assert.Equal(t, err.Error(), "bucket name b len is between [3-63],now is 1") assert.Nil(t, bucket) @@ -146,7 +149,7 @@ func TestPut(t *testing.T) { err = oss.Put(context.Background(), req) assert.Error(t, err) - req.FileName = "b/a.txt" + req.FileName = fileName err = oss.Put(context.Background(), req) assert.Error(t, err) } @@ -167,8 +170,8 @@ func TestGet(t *testing.T) { assert.Error(t, err) assert.Nil(t, resp) - req.FileName = "b/a.txt" - resp, err = oss.Get(context.Background(), req) + req.FileName = fileName + _, err = oss.Get(context.Background(), req) assert.Error(t, err) } @@ -188,8 +191,8 @@ func TestStat(t *testing.T) { assert.Error(t, err) assert.Nil(t, resp) - req.FileName = "b/a.txt" - resp, err = oss.Stat(context.Background(), req) + req.FileName = fileName + _, err = oss.Stat(context.Background(), req) assert.Error(t, err) } @@ -211,7 +214,7 @@ func TestList(t *testing.T) { assert.Nil(t, resp) req.DirectoryName = "b/" - resp, err = oss.List(context.Background(), req) + _, err = oss.List(context.Background(), req) assert.Error(t, err) } @@ -229,7 +232,7 @@ func TestDel(t *testing.T) { err = oss.Del(context.Background(), req) assert.Error(t, err) - req.FileName = "b/a.txt" + req.FileName = fileName err = oss.Del(context.Background(), req) assert.Error(t, err) } diff --git a/components/file/s3/aws/oss.go b/components/file/s3/aws/oss.go index 1aec174122..11a3ffe198 100644 --- a/components/file/s3/aws/oss.go +++ b/components/file/s3/aws/oss.go @@ -35,7 +35,6 @@ import ( const ( endpointKey = "endpoint" - bucketKey = "bucket" defaultCredentialsSource = "provider" ) diff --git a/components/file/s3/aws/oss_test.go b/components/file/s3/aws/oss_test.go index f761ab6caf..4a553dc91c 100644 --- a/components/file/s3/aws/oss_test.go +++ b/components/file/s3/aws/oss_test.go @@ -65,7 +65,7 @@ func TestAwsOss_SelectClient(t *testing.T) { meta["endpoint"] = "protocol://cn-northwest-1.region-code.amazonaws.com" client, err = oss.selectClient(meta) assert.Nil(t, err) - + assert.NotNil(t, client) // new client with endpoint oss.client["protocol://cn-northwest-1.region-code.amazonaws.com"] = &s3.Client{} client, _ = oss.selectClient(meta) @@ -97,6 +97,7 @@ func TestAwsOss_Put(t *testing.T) { req.FileName = "/a.txt" err = oss.Put(context.Background(), req) + assert.Equal(t, err.Error(), "awsoss put file[/a.txt] fail,err: invalid fileName format") } func TestAwsOss_Get(t *testing.T) { diff --git a/components/file/s3/minio/oss.go b/components/file/s3/minio/oss.go index 8f9415f4c9..50b27aee56 100644 --- a/components/file/s3/minio/oss.go +++ b/components/file/s3/minio/oss.go @@ -225,7 +225,7 @@ func (m *MinioOss) createOssClient(meta *MinioMetaData) (*minio.Core, error) { if err != nil { return nil, err } - return &minio.Core{client}, nil + return &minio.Core{Client: client}, nil } func (m *MinioOss) selectClient(meta map[string]string) (client *minio.Core, err error) { diff --git a/components/file/s3/qiniu/qiniu_oss_test.go b/components/file/s3/qiniu/qiniu_oss_test.go index 159209ed48..c4bd650305 100644 --- a/components/file/s3/qiniu/qiniu_oss_test.go +++ b/components/file/s3/qiniu/qiniu_oss_test.go @@ -31,15 +31,9 @@ import ( "mosn.io/layotto/components/pkg/mock" ) -func TestNew(t *testing.T) { - s := NewQiniuOSS() - assert.NotNil(t, s) -} - -func TestInit(t *testing.T) { - oss := NewQiniuOSS() - - data := `[ +const ( + endpointAddress = "endpoint_address" + metadata = `[ { "endpoint": "endpoint_address", "accessKeyID": "accessKey", @@ -47,7 +41,8 @@ func TestInit(t *testing.T) { "bucket": "xc2022" } ]` - data2 := `[ + + metadata2 = `[ { "endpoint": "endpoint_address", "accessKeyID": "accessKey", @@ -56,19 +51,28 @@ func TestInit(t *testing.T) { "useHTTPS": true } ]` +) + +func TestNew(t *testing.T) { + s := NewQiniuOSS() + assert.NotNil(t, s) +} + +func TestInit(t *testing.T) { + oss := NewQiniuOSS() fc := file.FileConfig{} err := oss.Init(context.Background(), &fc) assert.Error(t, err) - fc.Metadata = []byte(data) + fc.Metadata = []byte(metadata) err = oss.Init(context.Background(), &fc) assert.NoError(t, err) - fc.Metadata = []byte(data2) + fc.Metadata = []byte(metadata2) err = oss.Init(context.Background(), &fc) assert.Error(t, err) - fc.Metadata = []byte(data + ",") + fc.Metadata = []byte(metadata + ",") err = oss.Init(context.Background(), &fc) assert.Error(t, err) @@ -120,16 +124,8 @@ func TestCheckMetadata(t *testing.T) { func TestSelectClient(t *testing.T) { oss := NewQiniuOSS().(*QiniuOSS) - data := `[ - { - "endpoint": "endpoint_address", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "bucket": "xc2022" - } - ]` fc := file.FileConfig{} - fc.Metadata = []byte(data) + fc.Metadata = []byte(metadata) err := oss.Init(context.Background(), &fc) assert.NoError(t, err) @@ -144,7 +140,7 @@ func TestSelectClient(t *testing.T) { assert.Error(t, err) assert.Nil(t, client) - mt[endpointKey] = "endpoint_address" + mt[endpointKey] = endpointAddress client, err = oss.selectClient(mt) assert.NoError(t, err) assert.NotNil(t, client) @@ -180,16 +176,8 @@ func TestSelectClientWithMulti(t *testing.T) { func TestPut(t *testing.T) { oss := NewQiniuOSS() - data := `[ - { - "endpoint": "endpoint_address", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "bucket": "xc2022" - } - ]` fc := file.FileConfig{} - fc.Metadata = []byte(data) + fc.Metadata = []byte(metadata) err := oss.Init(context.Background(), &fc) assert.NoError(t, err) @@ -203,11 +191,11 @@ func TestPut(t *testing.T) { err = oss.Put(context.Background(), st) assert.Error(t, err) - st.Metadata[endpointKey] = "endpoint_address" + st.Metadata[endpointKey] = endpointAddress err = oss.Put(context.Background(), st) assert.Error(t, err) - st.Metadata[fileSizeKey] = "endpoint_address" + st.Metadata[fileSizeKey] = endpointAddress err = oss.Put(context.Background(), st) assert.Error(t, err) @@ -250,16 +238,8 @@ func TestGet(t *testing.T) { func TestDel(t *testing.T) { oss := NewQiniuOSS() - data := `[ - { - "endpoint": "endpoint_address", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "bucket": "xc2022" - } - ]` fc := file.FileConfig{} - fc.Metadata = []byte(data) + fc.Metadata = []byte(metadata) err := oss.Init(context.Background(), &fc) assert.NoError(t, err) @@ -272,23 +252,15 @@ func TestDel(t *testing.T) { err = oss.Del(context.Background(), st) assert.Error(t, err) - st.Metadata[endpointKey] = "endpoint_address" + st.Metadata[endpointKey] = endpointAddress err = oss.Del(context.Background(), st) assert.Error(t, err) } func TestStat(t *testing.T) { oss := NewQiniuOSS().(*QiniuOSS) - data := `[ - { - "endpoint": "endpoint_address", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "bucket": "xc2022" - } - ]` fc := file.FileConfig{} - fc.Metadata = []byte(data) + fc.Metadata = []byte(metadata) err := oss.Init(context.Background(), &fc) assert.NoError(t, err) @@ -303,7 +275,7 @@ func TestStat(t *testing.T) { assert.Error(t, err) assert.Nil(t, resp) - st.Metadata[endpointKey] = "endpoint_address" + st.Metadata[endpointKey] = endpointAddress resp, err = oss.Stat(context.Background(), st) assert.Error(t, err) assert.Nil(t, resp) @@ -316,7 +288,7 @@ func TestStat(t *testing.T) { bm.EXPECT().Stat(gomock.Eq("xc2022"), gomock.Eq("a.txt")).Return(storage.FileInfo{}, nil) mockOss(oss, bm, fu) - st.Metadata[endpointKey] = "endpoint_address" + st.Metadata[endpointKey] = endpointAddress resp, err = oss.Stat(context.Background(), st) assert.NoError(t, err) assert.NotNil(t, resp) @@ -325,16 +297,8 @@ func TestStat(t *testing.T) { func TestList(t *testing.T) { oss := NewQiniuOSS().(*QiniuOSS) - data := `[ - { - "endpoint": "endpoint_address", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "bucket": "xc2022" - } - ]` fc := file.FileConfig{} - fc.Metadata = []byte(data) + fc.Metadata = []byte(metadata) err := oss.Init(context.Background(), &fc) assert.NoError(t, err) @@ -350,7 +314,7 @@ func TestList(t *testing.T) { assert.Error(t, err) assert.Nil(t, resp) - st.Metadata[endpointKey] = "endpoint_address" + st.Metadata[endpointKey] = endpointAddress resp, err = oss.List(context.Background(), st) assert.Error(t, err) assert.Nil(t, resp) @@ -371,7 +335,7 @@ func TestList(t *testing.T) { bm.EXPECT().ListFiles(gomock.Eq("xc2022"), gomock.Eq("b/"), gomock.Any(), gomock.Any(), gomock.Any()).Return(items, make([]string, 0), "", false, nil) mockOss(oss, bm, fu) - st.Metadata[endpointKey] = "endpoint_address" + st.Metadata[endpointKey] = endpointAddress resp, err = oss.List(context.Background(), st) assert.NoError(t, err) assert.NotNil(t, resp) diff --git a/components/file/s3/tencentcloud/oss.go b/components/file/s3/tencentcloud/oss.go index da6052aa8b..b833fda35a 100644 --- a/components/file/s3/tencentcloud/oss.go +++ b/components/file/s3/tencentcloud/oss.go @@ -93,12 +93,11 @@ func (t *TencentCloudOSS) checkMetadata(m *OssMetadata) bool { if !strings.HasPrefix(m.Endpoint, "http") { endpoint = "https://" + m.Endpoint } - - if bucketUrl, err := url.Parse(endpoint); err != nil { + bucketUrl, err := url.Parse(endpoint) + if err != nil { return false - } else { - m.bucketUrl = bucketUrl } + m.bucketUrl = bucketUrl if m.Timeout <= 0 { m.Timeout = 100 * 1000 //100s diff --git a/components/file/s3/tencentcloud/oss_test.go b/components/file/s3/tencentcloud/oss_test.go index 27c8a9ba16..54d95d0c24 100644 --- a/components/file/s3/tencentcloud/oss_test.go +++ b/components/file/s3/tencentcloud/oss_test.go @@ -28,6 +28,15 @@ import ( "mosn.io/layotto/components/file" ) +const data = `[ + { + "endpoint": "https://xxx-1251058690.cos.ap-chengdu.myqcloud.com", + "accessKeyID": "accessKey", + "accessKeySecret": "secret", + "timeout": 1000 + } + ]` + func TestInit(t *testing.T) { data := `[ { @@ -160,14 +169,6 @@ func TestSelectClient(t *testing.T) { } func TestPut(t *testing.T) { - data := `[ - { - "endpoint": "https://xxx-1251058690.cos.ap-chengdu.myqcloud.com", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "timeout": 1000 - } - ]` fc := file.FileConfig{} oss := NewTencentCloudOSS() fc.Metadata = []byte(data) @@ -188,14 +189,6 @@ func TestPut(t *testing.T) { } func TestStat(t *testing.T) { - data := `[ - { - "endpoint": "https://xxx-1251058690.cos.ap-chengdu.myqcloud.com", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "timeout": 1000 - } - ]` fc := file.FileConfig{} oss := NewTencentCloudOSS() fc.Metadata = []byte(data) @@ -214,14 +207,6 @@ func TestStat(t *testing.T) { } func TestList(t *testing.T) { - data := `[ - { - "endpoint": "https://xxx-1251058690.cos.ap-chengdu.myqcloud.com", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "timeout": 1000 - } - ]` fc := file.FileConfig{} oss := NewTencentCloudOSS() fc.Metadata = []byte(data) @@ -252,14 +237,6 @@ func TestList(t *testing.T) { } func TestGet(t *testing.T) { - data := `[ - { - "endpoint": "https://xxx-1251058690.cos.ap-chengdu.myqcloud.com", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "timeout": 1000 - } - ]` fc := file.FileConfig{} oss := NewTencentCloudOSS() fc.Metadata = []byte(data) @@ -277,14 +254,6 @@ func TestGet(t *testing.T) { } func TestDel(t *testing.T) { - data := `[ - { - "endpoint": "https://xxx-1251058690.cos.ap-chengdu.myqcloud.com", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "timeout": 1000 - } - ]` fc := file.FileConfig{} oss := NewTencentCloudOSS() fc.Metadata = []byte(data) @@ -300,14 +269,6 @@ func TestDel(t *testing.T) { } func TestCheckFileName(t *testing.T) { - data := `[ - { - "endpoint": "https://xxx-1251058690.cos.ap-chengdu.myqcloud.com", - "accessKeyID": "accessKey", - "accessKeySecret": "secret", - "timeout": 1000 - } - ]` fc := file.FileConfig{} oss := NewTencentCloudOSS() fc.Metadata = []byte(data) diff --git a/components/lock/consul/consul_lock.go b/components/lock/consul/consul_lock.go index a6be627849..a88e44403a 100644 --- a/components/lock/consul/consul_lock.go +++ b/components/lock/consul/consul_lock.go @@ -51,6 +51,9 @@ func (c *ConsulLock) Init(metadata lock.Metadata) error { Address: consulMetadata.Address, Scheme: consulMetadata.Scheme, }) + if err != nil { + return err + } c.client = client c.sessionFactory = client.Session() c.kv = client.KV() @@ -98,12 +101,10 @@ func (c *ConsulLock) TryLock(req *lock.TryLockRequest) (*lock.TryLockResponse, e return &lock.TryLockResponse{ Success: true, }, nil - } else { - return &lock.TryLockResponse{ - Success: false, - }, nil } - + return &lock.TryLockResponse{ + Success: false, + }, nil } func (c *ConsulLock) Unlock(req *lock.UnlockRequest) (*lock.UnlockResponse, error) { @@ -129,7 +130,6 @@ func (c *ConsulLock) Unlock(req *lock.UnlockRequest) (*lock.UnlockResponse, erro } return &lock.UnlockResponse{Status: lock. SUCCESS}, nil - } else { - return &lock.UnlockResponse{Status: lock.LOCK_BELONG_TO_OTHERS}, nil } + return &lock.UnlockResponse{Status: lock.LOCK_BELONG_TO_OTHERS}, nil } diff --git a/components/lock/consul/consul_lock_test.go b/components/lock/consul/consul_lock_test.go index 82c0c102ce..84468deeec 100644 --- a/components/lock/consul/consul_lock_test.go +++ b/components/lock/consul/consul_lock_test.go @@ -63,6 +63,7 @@ func TestConsulLock_TryLock(t *testing.T) { } cfg.Properties["address"] = "127.0.0.1:8500" err := comp.Init(cfg) + assert.Nil(t, err) comp.client = client comp.sessionFactory = factory comp.kv = kv diff --git a/components/lock/etcd/etcd_lock.go b/components/lock/etcd/etcd_lock.go index 41a2f2296e..65e713111d 100644 --- a/components/lock/etcd/etcd_lock.go +++ b/components/lock/etcd/etcd_lock.go @@ -76,11 +76,11 @@ func (e *EtcdLock) TryLock(req *lock.TryLockRequest) (*lock.TryLockResponse, err var leaseId clientv3.LeaseID //1.Create new lease lease := clientv3.NewLease(e.client) - if leaseGrantResp, err := lease.Grant(e.ctx, int64(req.Expire)); err != nil { + leaseGrantResp, err := lease.Grant(e.ctx, int64(req.Expire)) + if err != nil { return &lock.TryLockResponse{}, fmt.Errorf("[etcdLock]: Create new lease returned error: %s.ResourceId: %s", err, req.ResourceId) - } else { - leaseId = leaseGrantResp.ID } + leaseId = leaseGrantResp.ID key := e.getKey(req.ResourceId) @@ -121,14 +121,13 @@ func (e *EtcdLock) Unlock(req *lock.UnlockRequest) (*lock.UnlockResponse, error) if txnResponse.Succeeded { return &lock.UnlockResponse{Status: lock.SUCCESS}, nil - } else { - resp := txnResponse.Responses[0].GetResponseRange() - if len(resp.Kvs) == 0 { - return &lock.UnlockResponse{Status: lock.LOCK_UNEXIST}, nil - } - - return &lock.UnlockResponse{Status: lock.LOCK_BELONG_TO_OTHERS}, nil } + resp := txnResponse.Responses[0].GetResponseRange() + if len(resp.Kvs) == 0 { + return &lock.UnlockResponse{Status: lock.LOCK_UNEXIST}, nil + } + + return &lock.UnlockResponse{Status: lock.LOCK_BELONG_TO_OTHERS}, nil } // Close shuts down the client's etcd connections. diff --git a/components/lock/mongo/mongo_lock.go b/components/lock/mongo/mongo_lock.go index 1ff489431a..ae0bfa44ce 100644 --- a/components/lock/mongo/mongo_lock.go +++ b/components/lock/mongo/mongo_lock.go @@ -94,7 +94,7 @@ func (e *MongoLock) Init(metadata lock.Metadata) error { // create exprie time index indexModel := mongo.IndexModel{ - Keys: bsonx.Doc{{"Expire", bsonx.Int64(1)}}, + Keys: bsonx.Doc{{Key: "Expire", Value: bsonx.Int64(1)}}, Options: options.Index().SetExpireAfterSeconds(0), } e.collection.Indexes().CreateOne(e.ctx, indexModel) @@ -161,11 +161,10 @@ func (e *MongoLock) TryLock(req *lock.TryLockRequest) (*lock.TryLockResponse, er return &lock.TryLockResponse{ Success: true, }, nil - } else { - return &lock.TryLockResponse{ - Success: false, - }, nil } + return &lock.TryLockResponse{ + Success: false, + }, nil } func (e *MongoLock) Unlock(req *lock.UnlockRequest) (*lock.UnlockResponse, error) { diff --git a/components/lock/mongo/mongo_lock_test.go b/components/lock/mongo/mongo_lock_test.go index 132de79a4b..d65b4a6aed 100644 --- a/components/lock/mongo/mongo_lock_test.go +++ b/components/lock/mongo/mongo_lock_test.go @@ -143,8 +143,7 @@ func TestMongoLock_Unlock(t *testing.T) { } cfg.Properties["mongoHost"] = mongoUrl - err = comp.Init(cfg) - + _ = comp.Init(cfg) // mock insertManyResult := &mongo.InsertManyResult{} insertOneResult := &mongo.InsertOneResult{} diff --git a/components/lock/redis/cluster_redis_lock.go b/components/lock/redis/cluster_redis_lock.go index 0aeb20d060..a9f9e62f91 100644 --- a/components/lock/redis/cluster_redis_lock.go +++ b/components/lock/redis/cluster_redis_lock.go @@ -129,25 +129,24 @@ func (c *ClusterRedisLock) TryLock(req *lock.TryLockRequest) (*lock.TryLockRespo if len(errorStrs) > 0 { err = fmt.Errorf(strings.Join(errorStrs, "\n")) } - //getting lock on majority of redis cluster will be regarded as locking success if successCount*2 > len(c.clients) { return &lock.TryLockResponse{ Success: true, }, err - } else { - _, unlockErr := c.UnlockAllRedis(&lock.UnlockRequest{ - ResourceId: req.ResourceId, - LockOwner: req.LockOwner, - }, &wg) - if unlockErr != nil { - errorStrs = append(errorStrs, unlockErr.Error()) - err = fmt.Errorf(strings.Join(errorStrs, "\n")) - } - return &lock.TryLockResponse{ - Success: false, - }, err } + + _, unlockErr := c.UnlockAllRedis(&lock.UnlockRequest{ + ResourceId: req.ResourceId, + LockOwner: req.LockOwner, + }, &wg) + if unlockErr != nil { + errorStrs = append(errorStrs, unlockErr.Error()) + err = fmt.Errorf(strings.Join(errorStrs, "\n")) + } + return &lock.TryLockResponse{ + Success: false, + }, err } func (c *ClusterRedisLock) Unlock(req *lock.UnlockRequest) (*lock.UnlockResponse, error) { diff --git a/components/lock/redis/cluster_redis_lock_test.go b/components/lock/redis/cluster_redis_lock_test.go index 3f028f965e..4b8fdd4110 100644 --- a/components/lock/redis/cluster_redis_lock_test.go +++ b/components/lock/redis/cluster_redis_lock_test.go @@ -26,7 +26,10 @@ import ( "mosn.io/layotto/components/lock" ) -const cResourceId = "resource_red_lock" +const ( + redisHosts = "127.0.0.1" + cResourceId = "resource_red_lock" +) func TestClusterRedisLock_InitError(t *testing.T) { t.Run("error when connection fail", func(t *testing.T) { @@ -36,7 +39,7 @@ func TestClusterRedisLock_InitError(t *testing.T) { cfg := lock.Metadata{ Properties: make(map[string]string), } - cfg.Properties["redisHosts"] = "127.0.0.1" + cfg.Properties["redisHosts"] = redisHosts cfg.Properties["redisPassword"] = "" // init @@ -66,7 +69,7 @@ func TestClusterRedisLock_InitError(t *testing.T) { cfg := lock.Metadata{ Properties: make(map[string]string), } - cfg.Properties["redisHosts"] = "127.0.0.1" + cfg.Properties["redisHosts"] = redisHosts cfg.Properties["redisPassword"] = "" cfg.Properties["maxRetries"] = "1 " @@ -79,13 +82,11 @@ func TestClusterRedisLock_InitError(t *testing.T) { func TestClusterRedisLock_TryLock(t *testing.T) { // start 5 miniredis instances - redisInstances := make([]*miniredis.Miniredis, 0, 5) redisAddrs := make([]string, 0, 5) var err error for i := 0; i < 5; i++ { redis, err := miniredis.Run() assert.NoError(t, err) - redisInstances = append(redisInstances, redis) redisAddrs = append(redisAddrs, redis.Addr()) } // construct component diff --git a/components/lock/zookeeper/zookeeper_lock_test.go b/components/lock/zookeeper/zookeeper_lock_test.go index a02117f3f3..4246110f03 100644 --- a/components/lock/zookeeper/zookeeper_lock_test.go +++ b/components/lock/zookeeper/zookeeper_lock_test.go @@ -14,6 +14,7 @@ package zookeeper import ( + "os" "testing" "time" @@ -39,7 +40,7 @@ func TestMain(m *testing.M) { cfg.Properties["zookeeperHosts"] = "127.0.0.1;127.0.0.1" cfg.Properties["zookeeperPassword"] = "" - m.Run() + os.Exit(m.Run()) } diff --git a/components/pkg/common/error.go b/components/pkg/common/error.go index 27c1616450..7bd9a73b56 100644 --- a/components/pkg/common/error.go +++ b/components/pkg/common/error.go @@ -34,8 +34,8 @@ type CommonError interface { } type commonError struct { - code int `json:"code"` - msg string `json:"msg"` + code int + msg string } func (le *commonError) Code() int { diff --git a/components/pkg/utils/etcd.go b/components/pkg/utils/etcd.go index 8def2a1e48..a16a359d3a 100644 --- a/components/pkg/utils/etcd.go +++ b/components/pkg/utils/etcd.go @@ -141,18 +141,17 @@ func NewEtcdClient(meta EtcdMetadata) (*clientv3.Client, error) { } config.TLS = tlsConfig } - - if client, err := clientv3.New(config); err != nil { + client, err := clientv3.New(config) + if err != nil { return nil, err - } else { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(meta.DialTimeout)) - defer cancel() - //ping - _, err = client.Get(ctx, "ping") - if err != nil { - return nil, fmt.Errorf("etcd error: connect to etcd timeoout %s", meta.Endpoints) - } - - return client, nil } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(meta.DialTimeout)) + defer cancel() + //ping + _, err = client.Get(ctx, "ping") + if err != nil { + return nil, fmt.Errorf("etcd error: connect to etcd timeoout %s", meta.Endpoints) + } + + return client, nil } diff --git a/components/pkg/utils/mongo.go b/components/pkg/utils/mongo.go index a29b75c102..ec7110f5ca 100644 --- a/components/pkg/utils/mongo.go +++ b/components/pkg/utils/mongo.go @@ -145,13 +145,9 @@ func (c *MongoFactoryImpl) NewMongoClient(m MongoMetadata) (MongoClient, error) func ParseMongoMetadata(properties map[string]string) (MongoMetadata, error) { m := MongoMetadata{} - if val, ok := properties[mongoHost]; ok && val != "" { - m.Host = val - } + m.Host = getString(properties, mongoHost) - if val, ok := properties[server]; ok && val != "" { - m.Server = val - } + m.Server = getString(properties, server) if len(m.Host) == 0 && len(m.Server) == 0 { return m, errors.New("must set 'host' or 'server' fields") @@ -161,13 +157,8 @@ func ParseMongoMetadata(properties map[string]string) (MongoMetadata, error) { return m, errors.New("'host' or 'server' fields are mutually exclusive") } - if val, ok := properties[username]; ok && val != "" { - m.Username = val - } - - if val, ok := properties[mongoPassword]; ok && val != "" { - m.Password = val - } + m.Username = getString(properties, username) + m.Password = getString(properties, mongoPassword) m.DatabaseName = defaultDatabase if val, ok := properties[databaseName]; ok && val != "" { @@ -179,17 +170,9 @@ func ParseMongoMetadata(properties map[string]string) (MongoMetadata, error) { m.CollectionName = val } - if val, ok := properties[writeConcern]; ok && val != "" { - m.WriteConcern = val - } - - if val, ok := properties[readConcern]; ok && val != "" { - m.ReadConcern = val - } - - if val, ok := properties[params]; ok && val != "" { - m.Params = val - } + m.WriteConcern = getString(properties, writeConcern) + m.ReadConcern = getString(properties, readConcern) + m.Params = getString(properties, params) var err error m.OperationTimeout = defaultTimeout @@ -202,6 +185,13 @@ func ParseMongoMetadata(properties map[string]string) (MongoMetadata, error) { return m, nil } +func getString(properties map[string]string, key string) string { + if val, ok := properties[key]; ok && val != "" { + return val + } + return "" +} + func getMongoURI(m MongoMetadata) string { if len(m.Server) != 0 { return fmt.Sprintf(connectionURIFormatWithSrv, m.Server, m.Params) diff --git a/components/pkg/utils/redis.go b/components/pkg/utils/redis.go index 9b089793a0..687a2f3c35 100644 --- a/components/pkg/utils/redis.go +++ b/components/pkg/utils/redis.go @@ -28,7 +28,7 @@ import ( const ( db = "db" host = "redisHost" - hosts = "redisHosts" + redisHosts = "redisHosts" password = "redisPassword" enableTLS = "enableTLS" maxRetries = "maxRetries" @@ -151,12 +151,12 @@ type RedisClusterMetadata struct { func ParseRedisClusterMetadata(properties map[string]string) (RedisClusterMetadata, error) { m := RedisClusterMetadata{} - if val, ok := properties[hosts]; ok && val != "" { - hosts := strings.Split(val, ",") - m.Hosts = hosts - } else { - return m, errors.New("redis store error: missing hosts address") + val, ok := properties[redisHosts] + if !ok || val == "" { + return m, errors.New("redis store error: missing redisHosts address") } + hosts := strings.Split(val, ",") + m.Hosts = hosts if val, ok := properties[password]; ok && val != "" { m.Password = val @@ -189,28 +189,35 @@ func ParseRedisClusterMetadata(properties map[string]string) (RedisClusterMetada m.MaxRetryBackoff = time.Duration(parsedVal) } + m.DB = defaultDB if val, ok := properties[db]; ok && val != "" { parsedVal, err := strconv.Atoi(val) if err != nil { return m, fmt.Errorf("redis store error: can't parse db field: %s", err) } m.DB = parsedVal - } else { - m.DB = defaultDB } + + con, err := getConcurrency(properties) + if err != nil { + return m, err + } + m.Concurrency = con + return m, nil +} + +func getConcurrency(properties map[string]string) (int, error) { + result := runtime.NumCPU() if val, ok := properties[concurrency]; ok && val != "" { con, err := strconv.Atoi(val) if err != nil { - return m, fmt.Errorf("redis store error: can't parse concurrency field: %s", err) + return result, fmt.Errorf("redis store error: can't parse concurrency field: %s", err) } - if con <= 0 { - con = runtime.NumCPU() + if con > 0 { + result = con } - m.Concurrency = con - } else { - m.Concurrency = runtime.NumCPU() } - return m, nil + return result, nil } func GetMiliTimestamp(i int64) int64 { diff --git a/components/rpc/invoker/mosn/channel/httpchannel.go b/components/rpc/invoker/mosn/channel/httpchannel.go index 5e2132defd..048ba48776 100644 --- a/components/rpc/invoker/mosn/channel/httpchannel.go +++ b/components/rpc/invoker/mosn/channel/httpchannel.go @@ -26,6 +26,7 @@ import ( "mosn.io/pkg/buffer" "github.com/valyala/fasthttp" + // bridge to mosn _ "mosn.io/mosn/pkg/stream/http" "mosn.io/layotto/components/pkg/common" diff --git a/components/rpc/invoker/mosn/channel/xchannel_test.go b/components/rpc/invoker/mosn/channel/xchannel_test.go index 1fe96e9b86..6056b26f3c 100644 --- a/components/rpc/invoker/mosn/channel/xchannel_test.go +++ b/components/rpc/invoker/mosn/channel/xchannel_test.go @@ -49,7 +49,7 @@ func (ts *testserver) accept(conn net.Conn, listener string) error { return nil } -func (s *testserver) handleRequest(frame api.XFrame) ([]byte, error) { +func (ts *testserver) handleRequest(frame api.XFrame) ([]byte, error) { data := frame.GetData() if data != nil { data := string(data.Bytes()) @@ -63,17 +63,17 @@ func (s *testserver) handleRequest(frame api.XFrame) ([]byte, error) { default: if strings.Contains(data, "echo") { resp := bolt.NewRpcResponse(uint32(frame.GetRequestId()), bolt.ResponseStatusSuccess, nil, buffer.NewIoBufferBytes([]byte(data))) - buf, _ := s.XProtocol.Encode(context.TODO(), resp) + buf, _ := ts.XProtocol.Encode(context.TODO(), resp) return buf.Bytes(), nil } } } resp := bolt.NewRpcResponse(uint32(frame.GetRequestId()), bolt.ResponseStatusSuccess, nil, buffer.NewIoBufferBytes([]byte("ok"))) - buf, _ := s.XProtocol.Encode(context.TODO(), resp) + buf, _ := ts.XProtocol.Encode(context.TODO(), resp) return buf.Bytes(), nil } -func (s *testserver) readLoop(conn net.Conn) { +func (ts *testserver) readLoop(conn net.Conn) { data := buffer.GetIoBuffer(1024) defer conn.Close() @@ -89,7 +89,7 @@ func (s *testserver) readLoop(conn net.Conn) { data.Write(p[:n]) for { - packet, err := s.XProtocol.Decode(context.TODO(), data) + packet, err := ts.XProtocol.Decode(context.TODO(), data) if err != nil { return } @@ -99,7 +99,7 @@ func (s *testserver) readLoop(conn net.Conn) { go func() { frame := packet.(*bolt.Request) - bytes, err := s.handleRequest(frame) + bytes, err := ts.handleRequest(frame) if err != nil { conn.Close() return @@ -255,7 +255,8 @@ func TestConncurrent(t *testing.T) { wg.Wait() req := &rpc.RPCRequest{Ctx: context.TODO(), Id: "foo", Method: "bar", Data: []byte("hello world"), Timeout: 1000} - _, err = channel.Do(req) + channel.Do(req) + //_, err = channel.Do(req) //assert.Nil(t, err) size = 100 diff --git a/components/rpc/invoker/mosn/mosninvoker.go b/components/rpc/invoker/mosn/mosninvoker.go index 5492d62adc..2917428ff6 100644 --- a/components/rpc/invoker/mosn/mosninvoker.go +++ b/components/rpc/invoker/mosn/mosninvoker.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" + // bridge to mosn _ "mosn.io/mosn/pkg/filter/network/proxy" "mosn.io/pkg/log" diff --git a/components/sequencer/etcd/store.go b/components/sequencer/etcd/store.go index 6dee7bad08..1887b5174c 100644 --- a/components/sequencer/etcd/store.go +++ b/components/sequencer/etcd/store.go @@ -107,7 +107,7 @@ func (e *EtcdSequencer) GetNextId(req *sequencer.GetNextIdRequest) (*sequencer.G }, nil } -func (s *EtcdSequencer) GetSegment(req *sequencer.GetSegmentRequest) (support bool, result *sequencer.GetSegmentResponse, err error) { +func (e *EtcdSequencer) GetSegment(req *sequencer.GetSegmentRequest) (support bool, result *sequencer.GetSegmentResponse, err error) { return false, nil, nil } diff --git a/components/sequencer/etcd/store_test.go b/components/sequencer/etcd/store_test.go index f6c4f906bf..f07a1ea06a 100644 --- a/components/sequencer/etcd/store_test.go +++ b/components/sequencer/etcd/store_test.go @@ -31,10 +31,6 @@ import ( ) const key = "resource_xxx" -const key2 = "resource_xxx2" - -const key3 = "resource_xxx3" -const key4 = "resource_xxx4" // GetFreePort returns a free port from the OS. func GetFreePort() (int, error) { @@ -157,6 +153,9 @@ func TestEtcd_GetNextId(t *testing.T) { expected = 2 assert.Equal(t, expected, resp.NextId) + support, _, err := comp.GetSegment(nil) + assert.False(t, support) + assert.Nil(t, err) } func startEtcdServer(dir string, port int) (*embed.Etcd, error) { diff --git a/components/trace/utils_test.go b/components/trace/utils_test.go index ea2a662676..4c3b226f6f 100644 --- a/components/trace/utils_test.go +++ b/components/trace/utils_test.go @@ -30,3 +30,19 @@ func TestSetExtraComponentInfo(t *testing.T) { v := span.Tag(LAYOTTO_COMPONENT_DETAIL) assert.Equal(t, v, "hello") } + +func TestSetterAndGetter(t *testing.T) { + var span Span + // ParentSpanId + span.SetParentSpanId("par") + v := span.ParentSpanId() + assert.Equal(t, v, "par") + // traceId + span.SetTraceId("traceId") + v = span.TraceId() + assert.Equal(t, v, "traceId") + // span id + span.SetSpanId("spanId") + v = span.SpanId() + assert.Equal(t, v, "spanId") +} diff --git a/demo/configuration/common/client.go b/demo/configuration/common/client.go index 84644a930a..fd2c0dcd9d 100644 --- a/demo/configuration/common/client.go +++ b/demo/configuration/common/client.go @@ -37,10 +37,14 @@ const ( writeTimes = 4 ) -var storeName string +var ( + storeName string + mode string +) func init() { flag.StringVar(&storeName, "s", "", "set `storeName`") + flag.StringVar(&mode, "mode", "raw", "set `mode`") } func main() { @@ -77,10 +81,13 @@ func main() { // 5. show how to use subscribe API // with sdk - //testSubscribeWithSDK(ctx, cli) + if mode == "sdk" { + testSubscribeWithSDK(ctx, cli) + } else { + // besides sdk,u can also call layotto with grpc + testSubscribeWithGrpc(ctx) + } - // besides sdk,u can also call layotto with grpc - testSubscribeWithGrpc(ctx) } func testSubscribeWithSDK(ctx context.Context, cli client.Client) { @@ -135,6 +142,9 @@ func testSubscribeWithGrpc(ctx context.Context) { var wg sync.WaitGroup wg.Add(2) cli, err := c.SubscribeConfiguration(ctx) + if err != nil { + panic(err) + } // client receive changes go func() { for { diff --git a/demo/faas/code/golang/go.mod b/demo/faas/code/golang/go.mod new file mode 100644 index 0000000000..b855e2ee92 --- /dev/null +++ b/demo/faas/code/golang/go.mod @@ -0,0 +1,12 @@ +module mosn.io/layotto/demo/faas/golang + +go 1.14 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/pretty v0.1.0 // indirect + github.com/tetratelabs/proxy-wasm-go-sdk v0.14.1-0.20210922004205-46e3ac3a25fe + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect +) + +replace github.com/tetratelabs/proxy-wasm-go-sdk => github.com/layotto/proxy-wasm-go-sdk v0.14.1-0.20210929091432-0e4ff35b75af diff --git a/demo/faas/code/golang/go.sum b/demo/faas/code/golang/go.sum new file mode 100644 index 0000000000..a7c0baa99b --- /dev/null +++ b/demo/faas/code/golang/go.sum @@ -0,0 +1,20 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/layotto/proxy-wasm-go-sdk v0.14.1-0.20210929091432-0e4ff35b75af h1:47gHlqE7EQXW2qRXWiAnogq8U7H+2VJUYxRLmCnVfsQ= +github.com/layotto/proxy-wasm-go-sdk v0.14.1-0.20210929091432-0e4ff35b75af/go.mod h1:qZ+4i6e2wHlhnhgpH0VG4QFzqd2BEvQbQFU0npt2e2k= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/demo/file/client.go b/demo/file/client.go index e89de29314..41264450e6 100644 --- a/demo/file/client.go +++ b/demo/file/client.go @@ -19,7 +19,8 @@ import ( ) const ( - storeName = "file_demo" + storeName = "file_demo" + storageType = "Standard" ) func TestGet(fileName string) { @@ -36,11 +37,14 @@ func TestGet(fileName string) { fmt.Printf("get file error: %+v", err) return } - pic := make([]byte, 0, 0) + pic := make([]byte, 0) for { resp, err := cli.Recv() if err != nil { - fmt.Errorf("recv file failed") + fmt.Println("recv file failed") + if err.Error() != "EOF" { + panic(err) + } break } pic = append(pic, resp.Data...) @@ -55,7 +59,7 @@ func TestPut(fileName string, value string) { return } meta := make(map[string]string) - meta["storageType"] = "Standard" + meta["storageType"] = storageType c := runtimev1pb.NewRuntimeClient(conn) req := &runtimev1pb.PutFileRequest{StoreName: storeName, Name: fileName, Metadata: meta} stream, err := c.PutFile(context.TODO()) @@ -78,7 +82,7 @@ func TestList(bucketName string) { return } meta := make(map[string]string) - meta["storageType"] = "Standard" + meta["storageType"] = storageType c := runtimev1pb.NewRuntimeClient(conn) marker := "" for { @@ -107,7 +111,7 @@ func TestDel(fileName string) { return } meta := make(map[string]string) - meta["storageType"] = "Standard" + meta["storageType"] = storageType c := runtimev1pb.NewRuntimeClient(conn) req := &runtimev1pb.FileRequest{StoreName: storeName, Name: fileName, Metadata: meta} listReq := &runtimev1pb.DelFileRequest{Request: req} @@ -126,7 +130,7 @@ func TestStat(fileName string) { return } meta := make(map[string]string) - meta["storageType"] = "Standard" + meta["storageType"] = storageType c := runtimev1pb.NewRuntimeClient(conn) req := &runtimev1pb.FileRequest{StoreName: storeName, Name: fileName, Metadata: meta} statReq := &runtimev1pb.GetFileMetaRequest{Request: req} @@ -136,11 +140,10 @@ func TestStat(fileName string) { if m.Code() == codes.NotFound { fmt.Println("file not exist") return - } else { - if m != nil { - fmt.Printf("stat file fail,err:%+v \n", err) - return - } + } + if m != nil { + fmt.Printf("stat file fail,err:%+v \n", err) + return } } fmt.Printf("get meta data of file: size:%+v, modifyTime:%+v \n", data.Size, data.LastModified) diff --git a/demo/file/qiniu/client.go b/demo/file/qiniu/client.go index c9f7867749..cef215df48 100644 --- a/demo/file/qiniu/client.go +++ b/demo/file/qiniu/client.go @@ -49,11 +49,11 @@ func TestGet(fileName string) { fmt.Printf("get file error: %+v", err) return } - pic := make([]byte, 0, 0) + pic := make([]byte, 0) for { resp, err := cli.Recv() if err != nil { - fmt.Errorf("recv file failed") + fmt.Printf("recv file failed") break } pic = append(pic, resp.Data...) @@ -150,11 +150,10 @@ func TestStat(fileName string) { if m.Code() == codes.NotFound { fmt.Println("file not exist") return - } else { - if m != nil { - fmt.Printf("stat file fail,err:%+v \n", err) - return - } + } + if m != nil { + fmt.Printf("stat file fail,err:%+v \n", err) + return } } fmt.Printf("get meta data of file: size:%+v, modifyTime:%+v \n", data.Size, data.LastModified) diff --git a/demo/file/stressmem.go b/demo/file/stressmem.go deleted file mode 100644 index b37e0434f8..0000000000 --- a/demo/file/stressmem.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "os" - "sync" - - "google.golang.org/grpc" - - runtimev1pb "mosn.io/layotto/spec/proto/runtime/v1" -) - -const ( - storeName1 = "file_demo" -) - -func GetFile(wg *sync.WaitGroup, id int) { - defer wg.Done() - conn, err := grpc.Dial("127.0.0.1:34904", grpc.WithInsecure()) - if err != nil { - fmt.Printf("conn build failed,err:%+v", err) - return - } - c := runtimev1pb.NewRuntimeClient(conn) - req := &runtimev1pb.GetFileRequest{StoreName: storeName1, Name: "fileName"} - cli, err := c.GetFile(context.Background(), req) - if err != nil { - fmt.Printf("get file error: %+v", err) - return - } - pic := make([]byte, 0, 0) - for { - resp, err := cli.Recv() - if err != nil { - fmt.Errorf("recv file failed") - break - } - pic = append(pic, resp.Data...) - } - ioutil.WriteFile("fileName", pic, os.ModePerm) - fmt.Printf("goroutine[%+v] finish get \n", id) -} - -func PutFile(wg *sync.WaitGroup, id int) { - defer wg.Done() - conn, err := grpc.Dial("127.0.0.1:34904", grpc.WithInsecure()) - if err != nil { - fmt.Printf("conn build failed,err:%+v", err) - return - } - meta := make(map[string]string) - meta["storageType"] = "Standard" - c := runtimev1pb.NewRuntimeClient(conn) - req := &runtimev1pb.PutFileRequest{StoreName: storeName1, Name: "fileName", Metadata: meta} - stream, err := c.PutFile(context.TODO()) - if err != nil { - fmt.Printf("put file failed:%+v", err) - return - } - fileHandle, err := os.Open("fileName") - defer fileHandle.Close() - //Upload in multiples, the minimum size is 100kb - buffer := make([]byte, 102400) - - for { - n, err := fileHandle.Read(buffer) - if err != nil && err != io.EOF { - fmt.Printf("read file failed, err:%+v", err) - break - } - if n == 0 { - //stream.CloseSend() - break - } - req.Data = buffer[:n] - err = stream.Send(req) - if err != nil { - fmt.Printf("send request failed: err: %+v", err) - break - } - } - _, err = stream.CloseAndRecv() - if err != nil { - fmt.Printf("cannot receive response: %+v", err) - return - } - fmt.Printf("goroutine[%+v] finish put \n", id) -} - -func main() { - var wg sync.WaitGroup - //Test when multi routine put big file, layotto memory cost - for i := 0; i < 100; i++ { - wg.Add(1) - PutFile(&wg, i) - } - //Test when multi routine get file, layotto memory cost - for { - for i := 0; i < 100; i++ { - wg.Add(1) - GetFile(&wg, i) - } - wg.Wait() - fmt.Println("finish test") - return - } -} diff --git a/demo/file/tencentcloud/client.go b/demo/file/tencentcloud/client.go index 05cd257f55..51cd5c77ed 100644 --- a/demo/file/tencentcloud/client.go +++ b/demo/file/tencentcloud/client.go @@ -48,11 +48,11 @@ func TestGet(fileName string) { fmt.Printf("get file error: %+v", err) return } - pic := make([]byte, 0, 0) + pic := make([]byte, 0) for { resp, err := cli.Recv() if err != nil { - fmt.Errorf("recv file failed") + fmt.Printf("recv file failed") break } pic = append(pic, resp.Data...) @@ -147,11 +147,10 @@ func TestStat(fileName string) { if m.Code() == codes.NotFound { fmt.Println("file not exist") return - } else { - if m != nil { - fmt.Printf("stat file fail,err:%+v \n", err) - return - } + } + if m != nil { + fmt.Printf("stat file fail,err:%+v \n", err) + return } } fmt.Printf("get meta data of file: size:%+v, modifyTime:%+v \n", data.Size, data.LastModified) diff --git a/demo/go.mod b/demo/go.mod index 549c6bb140..95562f2a67 100644 --- a/demo/go.mod +++ b/demo/go.mod @@ -6,15 +6,13 @@ require ( github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.3.0 github.com/minio/minio-go/v7 v7.0.15 - github.com/tetratelabs/proxy-wasm-go-sdk v0.14.1-0.20210922004205-46e3ac3a25fe - google.golang.org/grpc v1.38.0 + google.golang.org/grpc v1.37.0 google.golang.org/protobuf v1.26.0 mosn.io/layotto/sdk/go-sdk v0.0.0-20211020084508-6f5ee3cfeba0 mosn.io/layotto/spec v0.0.0-20211020084508-6f5ee3cfeba0 ) replace ( - github.com/tetratelabs/proxy-wasm-go-sdk => github.com/layotto/proxy-wasm-go-sdk v0.14.1-0.20210929091432-0e4ff35b75af mosn.io/layotto/sdk/go-sdk => ../sdk/go-sdk mosn.io/layotto/spec => ../spec ) diff --git a/demo/go.sum b/demo/go.sum index 31f0a75b75..49d9a91302 100644 --- a/demo/go.sum +++ b/demo/go.sum @@ -54,8 +54,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/layotto/proxy-wasm-go-sdk v0.14.1-0.20210929091432-0e4ff35b75af h1:47gHlqE7EQXW2qRXWiAnogq8U7H+2VJUYxRLmCnVfsQ= -github.com/layotto/proxy-wasm-go-sdk v0.14.1-0.20210929091432-0e4ff35b75af/go.mod h1:qZ+4i6e2wHlhnhgpH0VG4QFzqd2BEvQbQFU0npt2e2k= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v7 v7.0.15 h1:r9/NhjJ+nXYrIYvbObhvc1wPj3YH1iDpJzz61uRKLyY= @@ -138,9 +136,8 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/demo/lock/common/client.go b/demo/lock/common/client.go index 0356869358..914e177353 100644 --- a/demo/lock/common/client.go +++ b/demo/lock/common/client.go @@ -74,12 +74,9 @@ func main() { // 3. client 1 unlock fmt.Println("client1 prepare to unlock...") unlockResp, err := cli.Unlock(ctx, &runtimev1pb.UnlockRequest{ - StoreName: storeName, - ResourceId: resourceId, - LockOwner: owner1, - XXX_NoUnkeyedLiteral: struct{}{}, - XXX_unrecognized: nil, - XXX_sizecache: 0, + StoreName: storeName, + ResourceId: resourceId, + LockOwner: owner1, }) if err != nil { panic(err) @@ -108,12 +105,9 @@ func main() { fmt.Printf("client2 got lock.ResourceId is %s\n", resourceId) // 5. client2 unlock unlockResp, err := cli.Unlock(ctx, &runtimev1pb.UnlockRequest{ - StoreName: storeName, - ResourceId: resourceId, - LockOwner: owner2, - XXX_NoUnkeyedLiteral: struct{}{}, - XXX_unrecognized: nil, - XXX_sizecache: 0, + StoreName: storeName, + ResourceId: resourceId, + LockOwner: owner2, }) if err != nil { panic(err) diff --git a/demo/lock/zookeeper/client.go b/demo/lock/zookeeper/client.go index 7c38ab9519..8a1e2ffdda 100644 --- a/demo/lock/zookeeper/client.go +++ b/demo/lock/zookeeper/client.go @@ -64,12 +64,9 @@ func main() { // 3. client 1 unlock fmt.Println("client1 prepare to unlock...") unlockResp, err := cli.Unlock(ctx, &runtimev1pb.UnlockRequest{ - StoreName: storeName, - ResourceId: resourceId, - LockOwner: owner1, - XXX_NoUnkeyedLiteral: struct{}{}, - XXX_unrecognized: nil, - XXX_sizecache: 0, + StoreName: storeName, + ResourceId: resourceId, + LockOwner: owner1, }) if err != nil { panic(err) @@ -98,12 +95,9 @@ func main() { fmt.Printf("client2 got lock.ResourceId is %s\n", resourceId) // 5. client2 unlock unlockResp, err := cli.Unlock(ctx, &runtimev1pb.UnlockRequest{ - StoreName: storeName, - ResourceId: resourceId, - LockOwner: owner2, - XXX_NoUnkeyedLiteral: struct{}{}, - XXX_unrecognized: nil, - XXX_sizecache: 0, + StoreName: storeName, + ResourceId: resourceId, + LockOwner: owner2, }) if err != nil { panic(err) diff --git a/demo/pubsub/in-memory/in-memory.go b/demo/pubsub/in-memory/in-memory.go index 63e3da0512..4771255c5b 100644 --- a/demo/pubsub/in-memory/in-memory.go +++ b/demo/pubsub/in-memory/in-memory.go @@ -43,9 +43,4 @@ func testPublish(cli client.Client) { panic(err) } fmt.Printf("Published a new event.Topic: %s ,Data: %s \n", topic, data) - return -} - -func testSubscribe(cli client.Client) error { - return nil } diff --git a/docs/en/start/lock/start.md b/docs/en/start/lock/start.md index f1779819e8..0baabefb47 100644 --- a/docs/en/start/lock/start.md +++ b/docs/en/start/lock/start.md @@ -58,9 +58,9 @@ The layotto file will be generated in the directory, run it: ### Step 3. Run the client program, call Layotto to add, delete, modify and query ```shell - cd ${project_path}/demo/lock/redis/ + cd ${project_path}/demo/lock/common/ go build -o client - ./client + ./client -s "lock_demo" ``` If the following information is printed, the call is successful: diff --git a/docs/zh/start/lock/start.md b/docs/zh/start/lock/start.md index a8127c539e..3cd6581275 100644 --- a/docs/zh/start/lock/start.md +++ b/docs/zh/start/lock/start.md @@ -59,9 +59,9 @@ go build -o layotto ### 第三步:运行客户端程序,调用Layotto抢锁/解锁 ```shell - cd ${project_path}/demo/lock/redis/ + cd ${project_path}/demo/lock/common/ go build -o client - ./client + ./client -s "lock_demo" ``` 打印出如下信息则代表调用成功: diff --git a/etc/script/resolve-modules.sh b/etc/script/resolve-modules.sh new file mode 100644 index 0000000000..61966b87f6 --- /dev/null +++ b/etc/script/resolve-modules.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Recursively finds all directories with a go.mod file and creates +# a GitHub Actions JSON output option. This is used by the linter action. + +echo "Resolving modules in $(pwd)" + +PATHS=$(find . -not -path "*/faas/*" -type f -name go.mod -printf '{"workdir":"%h"},') +echo "::set-output name=matrix::{\"include\":[${PATHS%?}]}" \ No newline at end of file diff --git a/make/golang.mk b/make/golang.mk index ffb41f4d54..7fe0cfe100 100644 --- a/make/golang.mk +++ b/make/golang.mk @@ -91,7 +91,7 @@ endif .PHONY: go.lint go.lint: go.lint.verify @echo "===========> Run golangci to lint source codes" - @golangci-lint run + @golangci-lint run -v .PHONY: go.test.verify go.test.verify: diff --git a/sdk/go-sdk/client/client.go b/sdk/go-sdk/client/client.go index 4a12561c1b..7acafa55d9 100644 --- a/sdk/go-sdk/client/client.go +++ b/sdk/go-sdk/client/client.go @@ -174,7 +174,6 @@ func NewClientWithConnection(conn *grpc.ClientConn) Client { type GRPCClient struct { connection *grpc.ClientConn protoClient runtimev1pb.RuntimeClient - mux sync.Mutex } // Close cleans up all resources created by the client. diff --git a/sdk/go-sdk/client/secret.go b/sdk/go-sdk/client/secret.go index e5860d49e7..d4641a38ba 100644 --- a/sdk/go-sdk/client/secret.go +++ b/sdk/go-sdk/client/secret.go @@ -1,3 +1,4 @@ +// // Copyright 2021 Layotto Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -10,7 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - +// CODE ATTRIBUTION: https://github.com/dapr/go-sdk +// Modified the import package to use layotto's pb +// We use same sdk code with Dapr's for state API because we want to keep compatible with Dapr state API package client import ( diff --git a/sdk/go-sdk/client/state.go b/sdk/go-sdk/client/state.go index a70962309d..1589a05e1b 100755 --- a/sdk/go-sdk/client/state.go +++ b/sdk/go-sdk/client/state.go @@ -85,31 +85,31 @@ func (o OperationType) String() string { } // String returns the string value of the StateConsistency. -func (c StateConsistency) String() string { +func (s StateConsistency) String() string { names := [...]string{ UndefinedType, "strong", "eventual", } - if c < StateConsistencyStrong || c > StateConsistencyEventual { + if s < StateConsistencyStrong || s > StateConsistencyEventual { return UndefinedType } - return names[c] + return names[s] } // String returns the string value of the StateConcurrency. -func (c StateConcurrency) String() string { +func (s StateConcurrency) String() string { names := [...]string{ UndefinedType, "first-write", "last-write", } - if c < StateConcurrencyFirstWrite || c > StateConcurrencyLastWrite { + if s < StateConcurrencyFirstWrite || s > StateConcurrencyLastWrite { return UndefinedType } - return names[c] + return names[s] } var ( @@ -228,7 +228,7 @@ func toProtoDuration(d time.Duration) *duration.Duration { secs := nanos / 1e9 nanos -= secs * 1e9 return &duration.Duration{ - Seconds: int64(secs), + Seconds: secs, Nanos: int32(nanos), } } diff --git a/sdk/go-sdk/client/state_test.go b/sdk/go-sdk/client/state_test.go index b26ef008bd..5f0095b3d5 100755 --- a/sdk/go-sdk/client/state_test.go +++ b/sdk/go-sdk/client/state_test.go @@ -26,6 +26,8 @@ import ( v1 "mosn.io/layotto/spec/proto/runtime/v1" ) +const test = "test" + func TestTypes(t *testing.T) { var op OperationType = -1 assert.Equal(t, UndefinedType, op.String()) @@ -36,7 +38,7 @@ func TestTypes(t *testing.T) { } func TestDurationConverter(t *testing.T) { - d := time.Duration(10 * time.Second) + d := 10 * time.Second pd := toProtoDuration(d) assert.NotNil(t, pd) assert.Equal(t, pd.Seconds, int64(10)) @@ -56,8 +58,8 @@ func TestStateOptionsConverter(t *testing.T) { // go test -timeout 30s ./client -count 1 -run ^TestSaveState$ func TestSaveState(t *testing.T) { ctx := context.Background() - data := "test" - store := "test" + data := test + store := test key := "key1" t.Run("save data", func(t *testing.T) { @@ -104,8 +106,8 @@ func TestSaveState(t *testing.T) { // go test -timeout 30s ./client -count 1 -run ^TestDeleteState$ func TestDeleteState(t *testing.T) { ctx := context.Background() - data := "test" - store := "test" + data := test + store := test key := "key1" t.Run("delete not exist data", func(t *testing.T) { @@ -182,8 +184,8 @@ func TestDeleteState(t *testing.T) { func TestDeleteBulkState(t *testing.T) { ctx := context.Background() - data := "test" - store := "test" + data := test + store := test keys := []string{"key1", "key2", "key3"} t.Run("delete not exist data", func(t *testing.T) { @@ -290,7 +292,7 @@ func TestDeleteBulkState(t *testing.T) { func TestStateTransactions(t *testing.T) { ctx := context.Background() data := `{ "message": "test" }` - store := "test" + store := test meta := map[string]string{} keys := []string{"k1", "k2", "k3"} adds := make([]*StateOperation, 0)