diff --git a/configs/config_integrate_test.json b/configs/config_integrate_test.json new file mode 100644 index 0000000000..4e61790d83 --- /dev/null +++ b/configs/config_integrate_test.json @@ -0,0 +1,173 @@ +{ + "servers": [ + { + "default_log_path": "stdout", + "default_log_level": "DEBUG", + "routers": [ + { + "router_config_name": "actuator_dont_need_router" + } + ], + "listeners": [ + { + "name": "grpc", + "address": "127.0.0.1:34904", + "bind_port": true, + "filter_chains": [ + { + "filters": [ + { + "type": "grpc", + "config": { + "server_name": "runtime", + "grpc_config": { + "hellos": { + "quick_start_demo": { + "type": "helloworld", + "hello": "greeting" + } + }, + "config_store": { + "etcd_config_demo": { + "type": "etcd", + "address": [ + "127.0.0.1:2379" + ], + "timeout": "10" + } + }, + "state": { + "redis_state_demo": { + "type": "redis", + "metadata": { + "redisHost": "localhost:6380", + "redisPassword": "" + } + }, + "zookeeper_state_demo": { + "type": "zookeeper", + "metadata": { + "servers": "127.0.0.1", + "zookeeperPassword": "", + "sessionTimeout": "3s", + "logInfo": "false", + "keyPrefixPath": "/" + } + } + }, + "lock": { + "redis_lock_demo": { + "type": "redis", + "metadata": { + "redisHost": "localhost:6380", + "redisPassword": "" + } + }, + "etcd_lock_demo": { + "type": "etcd", + "metadata": { + "endpoints": "localhost:2379", + "username": "", + "password": "", + "keyPrefixPath": "/lock", + "dialTimeout": "5" + } + }, + "zookeeper_lock_demo": { + "type": "zookeeper", + "metadata": { + "zookeeperHosts": "127.0.0.1", + "zookeeperPassword": "", + "sessionTimeout": "3", + "logInfo": "false", + "keyPrefixPath": "/" + } + } + }, + "sequencer": { + "redis_sequencer_demo": { + "type": "redis", + "metadata": { + "redisHost": "127.0.0.1:6380", + "redisPassword": "" + } + }, + "etcd_sequencer_demo": { + "type": "etcd", + "metadata": { + "endpoints": "localhost:2379", + "segmentCacheEnable": "false", + "segmentStep": "1", + "username": "", + "password": "", + "dialTimeout": "5" + } + }, + "zookeeper_sequencer_demo": { + "type": "zookeeper", + "metadata": { + "zookeeperHosts": "127.0.0.1", + "zookeeperPassword": "", + "sessionTimeout": "3", + "logInfo": "false", + "keyPrefixPath": "/" + } + } + }, + "pub_subs": { + "redis_pub_subs_demo": { + "type": "redis", + "metadata": { + "redisHost": "localhost:6380", + "redisPassword": "" + } + } + }, + "secret_store": { + "local_file_secret_demo": { + "type": "local.file", + "metadata": { + "secretsFile": "./configs/secret/config_test_secret_demo.json" + } + } + }, + "app": { + "app_id": "app1", + "grpc_callback_port": 9999 + } + } + } + } + ] + } + ] + }, + { + "name": "actuator", + "address": "127.0.0.1:34999", + "bind_port": true, + "filter_chains": [ + { + "filters": [ + { + "type": "proxy", + "config": { + "downstream_protocol": "Http1", + "upstream_protocol": "Http1", + "router_config_name": "actuator_dont_need_router" + } + } + ] + } + ], + "stream_filters": [ + { + "type": "actuator_filter" + } + ] + } + ] + } + ] + } + \ No newline at end of file diff --git a/configs/secret/config_test_secret_demo.json b/configs/secret/config_test_secret_demo.json new file mode 100644 index 0000000000..e0f8722e3b --- /dev/null +++ b/configs/secret/config_test_secret_demo.json @@ -0,0 +1,7 @@ +{ + "db-user-pass": { + "username": "admin", + "password": "pw1" + }, + "testPassword": "pw2" + } \ No newline at end of file diff --git a/docker/app/integrate/Dockerfile b/docker/app/integrate/Dockerfile index 4299556983..0521627a05 100644 --- a/docker/app/integrate/Dockerfile +++ b/docker/app/integrate/Dockerfile @@ -1,3 +1,6 @@ FROM golang:1.16 -RUN apt-get update && apt-get install -y redis-server +RUN apt-get update && apt-get install -y redis-server \ + && apt-get install -y etcd-server \ + && apt-get install -y net-tools \ + && apt-get install -y zookeeper \ \ No newline at end of file diff --git a/sdk/go-sdk/test/runtime/integrate_test.go b/sdk/go-sdk/test/runtime/integrate_test.go new file mode 100644 index 0000000000..077bb0a1d9 --- /dev/null +++ b/sdk/go-sdk/test/runtime/integrate_test.go @@ -0,0 +1,292 @@ +/* + * Copyright 2021 Layotto Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + "context" + "fmt" + "net" + "sync" + "testing" + "time" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc" + + "mosn.io/layotto/sdk/go-sdk/client" + runtimev1pb "mosn.io/layotto/spec/proto/runtime/v1" +) + +const ( + appid = "testApplication" + group = "application" +) + +var PubsubStoreName string + +func TestHelloApi(t *testing.T) { + cli, err := client.NewClientWithAddress("127.0.0.1:34904") + if err != nil { + t.Fatal(err) + } + defer cli.Close() + + ctx := context.Background() + + helloReq := &client.SayHelloRequest{ + ServiceName: "quick_start_demo", + } + helloResp, err := cli.SayHello(ctx, helloReq) + assert.Nil(t, err) + assert.Equal(t, "greeting", helloResp.Hello) +} + +func TestConfigurationApi(t *testing.T) { + cli, err := client.NewClientWithAddress("127.0.0.1:34904") + if err != nil { + t.Fatal(err) + } + defer cli.Close() + + ctx := context.Background() + + item1 := &client.ConfigurationItem{Group: group, Label: "test", Key: "key1", Content: "value1"} + item2 := &client.ConfigurationItem{Group: group, Label: "test", Key: "key2", Content: "value2"} + + componentArray := [...]string{"etcd_config_demo"} + for _, storeName := range componentArray { + saveRequest := &client.SaveConfigurationRequest{StoreName: storeName, AppId: appid} + saveRequest.Items = append(saveRequest.Items, item1) + saveRequest.Items = append(saveRequest.Items, item2) + err = cli.SaveConfiguration(ctx, saveRequest) + assert.Nil(t, err) + + // Since configuration data might be cached and eventual-consistent,we need to sleep a while before querying new data + time.Sleep(time.Second * 2) + getRequest := &client.ConfigurationRequestItem{StoreName: storeName, AppId: appid, Group: group, Label: "test", Keys: []string{"key1", "key2"}} + resp, err := cli.GetConfiguration(ctx, getRequest) + assert.Nil(t, err) + assert.Equal(t, item1, resp[0]) + assert.Equal(t, item2, resp[1]) + } +} + +func TestStateApi(t *testing.T) { + cli, err := client.NewClientWithAddress("127.0.0.1:34904") + if err != nil { + t.Fatal(err) + } + defer cli.Close() + + ctx := context.Background() + + stateKey := "MyStateKey" + stateValue := []byte("Hello Layotto!") + + componentArray := [...]string{"redis_state_demo", "zookeeper_state_demo"} + for _, storeName := range componentArray { + err = cli.SaveState(ctx, storeName, stateKey, stateValue) + assert.Nil(t, err) + + stateResp, err := cli.GetState(ctx, storeName, stateKey) + assert.Nil(t, err) + assert.Equal(t, stateValue, stateResp.Value) + } +} + +func TestLockApi(t *testing.T) { + cli, err := client.NewClientWithAddress("127.0.0.1:34904") + if err != nil { + t.Fatal(err) + } + defer cli.Close() + + ctx := context.Background() + + owner1 := uuid.New().String() + owner2 := uuid.New().String() + resourceID := "MyLock" + + componentArray := [...]string{"redis_lock_demo", "etcd_lock_demo", "zookeeper_lock_demo"} + for _, storeName := range componentArray { + // 1. client1 tryLock + resp, err := cli.TryLock(ctx, &runtimev1pb.TryLockRequest{ + StoreName: storeName, + ResourceId: resourceID, + LockOwner: owner1, + Expire: 100000, + }) + assert.Nil(t, err) + assert.True(t, resp.Success) + + var wg sync.WaitGroup + wg.Add(1) + // 2. client2 tryLock + go func() { + resp, err := cli.TryLock(ctx, &runtimev1pb.TryLockRequest{ + StoreName: storeName, + ResourceId: resourceID, + LockOwner: owner2, + Expire: 1000, + }) + assert.Nil(t, err) + assert.False(t, resp.Success) + wg.Done() + }() + wg.Wait() + // 3. client1 unlock + unlockResp, err := cli.Unlock(ctx, &runtimev1pb.UnlockRequest{ + StoreName: storeName, + ResourceId: resourceID, + LockOwner: owner1, + }) + assert.Nil(t, err) + assert.Equal(t, runtimev1pb.UnlockResponse_SUCCESS, unlockResp.Status) + + // 4. client2 get lock + wg.Add(1) + go func() { + resp, err := cli.TryLock(ctx, &runtimev1pb.TryLockRequest{ + StoreName: storeName, + ResourceId: resourceID, + LockOwner: owner2, + Expire: 10, + }) + assert.Nil(t, err) + assert.True(t, true, resp.Success) + // 5. client2 unlock + unlockResp, err := cli.Unlock(ctx, &runtimev1pb.UnlockRequest{ + StoreName: storeName, + ResourceId: resourceID, + LockOwner: owner2, + }) + assert.Nil(t, err) + assert.Equal(t, runtimev1pb.UnlockResponse_SUCCESS, unlockResp.Status) + wg.Done() + }() + wg.Wait() + } +} + +func TestSequencerApi(t *testing.T) { + cli, err := client.NewClientWithAddress("127.0.0.1:34904") + if err != nil { + t.Fatal(err) + } + defer cli.Close() + + ctx := context.Background() + + sequencerKey := "MySequencerKey" + + componentArray := [...]string{"redis_sequencer_demo", "etcd_sequencer_demo", "zookeeper_sequencer_demo"} + for _, storeName := range componentArray { + for i := 1; i < 10; i++ { + resp, err := cli.GetNextId(ctx, &runtimev1pb.GetNextIdRequest{ + StoreName: storeName, + Key: sequencerKey, + }) + assert.Nil(t, err) + assert.Equal(t, int64(i), resp.NextId) + } + } +} + +func TestPubsubApi(t *testing.T) { + cli, err := client.NewClientWithAddress("127.0.0.1:34904") + if err != nil { + t.Fatal(err) + } + defer cli.Close() + + ctx := context.Background() + + componentArray := [...]string{"redis_pub_subs_demo"} + for _, storeName := range componentArray { + PubsubStoreName = storeName + //start a subscriber to subscribe to events + go func() { + port := 8888 + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + assert.Nil(t, err) + grpcServer := grpc.NewServer() + runtimev1pb.RegisterAppCallbackServer(grpcServer, &AppCallbackServerImpl{}) + err = grpcServer.Serve(lis) + assert.Nil(t, err) + }() + + //publisher publishes an event + err := cli.PublishEventfromCustomContent(ctx, storeName, "topic1", "value1") + assert.Nil(t, err) + + //sleep for a while before subscriber get messages + time.Sleep(time.Second) + } +} + +func TestSecretApi(t *testing.T) { + cli, err := client.NewClientWithAddress("127.0.0.1:34904") + if err != nil { + t.Fatal(err) + } + defer cli.Close() + + ctx := context.Background() + + componentArray := [...]string{"local_file_secret_demo"} + for _, storeName := range componentArray { + //get the secret + secret, err := cli.GetSecret(ctx, &runtimev1pb.GetSecretRequest{ + StoreName: storeName, + Key: "testPassword", + }) + assert.Nil(t, err) + assert.Equal(t, "pw2", secret.Data["testPassword"]) + + //get the bulk secret + bulkSecret, err := cli.GetBulkSecret(ctx, &runtimev1pb.GetBulkSecretRequest{ + StoreName: storeName, + }) + assert.Nil(t, err) + assert.Equal(t, "admin", bulkSecret.Data["db-user-pass:username"].Secrets["db-user-pass:username"]) + assert.Equal(t, "pw1", bulkSecret.Data["db-user-pass:password"].Secrets["db-user-pass:password"]) + assert.Equal(t, "pw2", bulkSecret.Data["testPassword"].Secrets["testPassword"]) + } +} + +type AppCallbackServerImpl struct { +} + +func (a *AppCallbackServerImpl) ListTopicSubscriptions(ctx context.Context, empty *empty.Empty) (*runtimev1pb.ListTopicSubscriptionsResponse, error) { + result := &runtimev1pb.ListTopicSubscriptionsResponse{} + ts := &runtimev1pb.TopicSubscription{ + PubsubName: PubsubStoreName, + Topic: "topic1", + Metadata: nil, + } + result.Subscriptions = append(result.Subscriptions, ts) + return result, nil +} + +func (a *AppCallbackServerImpl) OnTopicEvent(ctx context.Context, request *runtimev1pb.TopicEventRequest) (*runtimev1pb.TopicEventResponse, error) { + var t *testing.T + assert.Equal(t, "topic1", request.Topic) + assert.Equal(t, "value1", request.Data) + return &runtimev1pb.TopicEventResponse{}, nil +} diff --git a/sdk/go-sdk/test/runtime/integrate_test.sh b/sdk/go-sdk/test/runtime/integrate_test.sh index ff0625d000..2a41afb114 100644 --- a/sdk/go-sdk/test/runtime/integrate_test.sh +++ b/sdk/go-sdk/test/runtime/integrate_test.sh @@ -14,8 +14,68 @@ # limitations under the License. # -go build ./cmd/layotto +# fail fast +set -e + +# start storage systems, e.g. redis, zk, etcd nohup redis-server --port 6380 & -nohup ./layotto start -c ./configs/config_redis.json & +bash /usr/share/zookeeper/bin/zkServer.sh start +nohup bash /usr/share/zookeeper/bin/zkCli.sh < ./sdk/go-sdk/test/runtime/zkCreateZnode.sh +cd .. +nohup etcd & + +# build and run Layotto +cd layotto +go build ./cmd/layotto +nohup ./layotto start -c ./configs/config_integrate_test.json & + +# run integrate_test cd sdk/go-sdk/test/runtime -go test -p 1 -v ./... \ No newline at end of file +go test -p 1 -v ./... + +# run demos +cd ../../../../demo/configuration/common +go build -o client +names="etcd_config_demo" +for key in ${names}; do + ./client -s $key +done + +cd ../../state/common +go build -o client +names="redis_state_demo zookeeper_state_demo" +for key in ${names}; do + ./client -s $key +done + +cd ../../lock/common +go build -o client +names="redis_lock_demo etcd_lock_demo zookeeper_lock_demo" +for key in ${names}; do + ./client -s $key +done + +cd ../../sequencer/common +go build -o client +names="redis_sequencer_demo etcd_sequencer_demo zookeeper_sequencer_demo" +for key in ${names}; do + ./client -s $key +done + +cd ../../pubsub/server +names="redis_pub_subs_demo" +for key in ${names}; do + cd ../server + go build -o subscriber + ./subscriber -s $key & + cd ../client + go build -o publisher + ./publisher -s $key +done + +cd ../../secret/common +go build -o client +names="local_file_secret_demo" +for key in ${names}; do + ./client -s $key +done \ No newline at end of file diff --git a/sdk/go-sdk/test/runtime/other_component_integrate_test.go b/sdk/go-sdk/test/runtime/other_component_integrate_test.go deleted file mode 100644 index 88056cc308..0000000000 --- a/sdk/go-sdk/test/runtime/other_component_integrate_test.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2021 Layotto Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package runtime - -import ( - "testing" -) - -func TestAllApiWithOtherComponents(t *testing.T) { - - //======================= Other Component Start ======================= - // TODO - //======================= Other Component End ======================= - -} diff --git a/sdk/go-sdk/test/runtime/redis_integrate_test.go b/sdk/go-sdk/test/runtime/redis_integrate_test.go deleted file mode 100644 index 894efcf3cf..0000000000 --- a/sdk/go-sdk/test/runtime/redis_integrate_test.go +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright 2021 Layotto Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package runtime - -import ( - "context" - "sync" - "testing" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - - "mosn.io/layotto/sdk/go-sdk/client" - runtimev1pb "mosn.io/layotto/spec/proto/runtime/v1" -) - -var componentName = "state_demo" - -func TestHelloApi(t *testing.T) { - cli, err := client.NewClientWithAddress("127.0.0.1:34904") - if err != nil { - t.Fatal(err) - } - defer cli.Close() - - ctx := context.Background() - - helloReq := &client.SayHelloRequest{ - ServiceName: "helloworld", - } - helloResp, err := cli.SayHello(ctx, helloReq) - assert.Nil(t, err) - assert.Equal(t, "greeting", helloResp.Hello) -} - -func TestStateApi(t *testing.T) { - cli, err := client.NewClientWithAddress("127.0.0.1:34904") - if err != nil { - t.Fatal(err) - } - defer cli.Close() - - ctx := context.Background() - - stateKey := "MyKey" - stateValue := []byte("Hello Layotto!") - err = cli.SaveState(ctx, componentName, stateKey, stateValue) - assert.Nil(t, err) - - stateResp, err := cli.GetState(ctx, "state_demo", stateKey) - assert.Nil(t, err) - assert.Equal(t, stateValue, stateResp.Value) -} - -func TestLockApi(t *testing.T) { - cli, err := client.NewClientWithAddress("127.0.0.1:34904") - if err != nil { - t.Fatal(err) - } - defer cli.Close() - - ctx := context.Background() - - owner1 := uuid.New().String() - owner2 := uuid.New().String() - resourceID := "MyLock" - storeName := "lock_demo" - // 1. client1 tryLock - resp, err := cli.TryLock(ctx, &runtimev1pb.TryLockRequest{ - StoreName: storeName, - ResourceId: resourceID, - LockOwner: owner1, - Expire: 100000, - }) - assert.Nil(t, err) - assert.True(t, resp.Success) - - var wg sync.WaitGroup - wg.Add(1) - // 2. client2 tryLock - go func() { - resp, err := cli.TryLock(ctx, &runtimev1pb.TryLockRequest{ - StoreName: storeName, - ResourceId: resourceID, - LockOwner: owner2, - Expire: 1000, - }) - assert.Nil(t, err) - assert.False(t, resp.Success) - wg.Done() - }() - wg.Wait() - // 3. client1 unlock - unlockResp, err := cli.Unlock(ctx, &runtimev1pb.UnlockRequest{ - StoreName: storeName, - ResourceId: resourceID, - LockOwner: owner1, - }) - assert.Nil(t, err) - assert.Equal(t, runtimev1pb.UnlockResponse_SUCCESS, unlockResp.Status) - - // 4. client2 get lock - wg.Add(1) - go func() { - resp, err := cli.TryLock(ctx, &runtimev1pb.TryLockRequest{ - StoreName: storeName, - ResourceId: resourceID, - LockOwner: owner2, - Expire: 10, - }) - assert.Nil(t, err) - assert.True(t, true, resp.Success) - // 5. client2 unlock - unlockResp, err := cli.Unlock(ctx, &runtimev1pb.UnlockRequest{ - StoreName: storeName, - ResourceId: resourceID, - LockOwner: owner2, - }) - assert.Nil(t, err) - assert.Equal(t, runtimev1pb.UnlockResponse_SUCCESS, unlockResp.Status) - wg.Done() - }() - wg.Wait() -} - -func TestSequencerApi(t *testing.T) { - cli, err := client.NewClientWithAddress("127.0.0.1:34904") - if err != nil { - t.Fatal(err) - } - defer cli.Close() - - ctx := context.Background() - sequencerKey := "MyKey" - - for i := 1; i < 10; i++ { - resp, err := cli.GetNextId(ctx, &runtimev1pb.GetNextIdRequest{ - StoreName: "sequencer_demo", - Key: sequencerKey, - }) - assert.Nil(t, err) - assert.Equal(t, int64(i), resp.NextId) - } -} diff --git a/sdk/go-sdk/test/runtime/zkCreateZnode.sh b/sdk/go-sdk/test/runtime/zkCreateZnode.sh new file mode 100644 index 0000000000..20927512f6 --- /dev/null +++ b/sdk/go-sdk/test/runtime/zkCreateZnode.sh @@ -0,0 +1,26 @@ + # + # Copyright 2021 Layotto Authors + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + +# sdk/go-sdk/test/runtime/integrate_test.go required +create /MyStateKey "" + +# demo/state/ required +create /key1 "" +create /key2 "" +create /key3 "" +create /key4 "" +create /key5 "" + +quit \ No newline at end of file