debug: debug

Signed-off-by: Gyu-Ho Lee <[email protected]>

.semaphore.sh

@@ -2,9 +2,9 @@
 
 TEST_SUFFIX=$(date +%s | base64 | head -c 15)
 
-TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional'"
+TEST_OPTS="PASSES='build integration_e2e'"
 if [ "$TEST_ARCH" == "386" ]; then
-	TEST_OPTS="GOARCH=386 PASSES='build unit integration_e2e'"
+	TEST_OPTS="GOARCH=386 PASSES='build integration_e2e'"
 fi
 
 docker run \

clientv3/balancer.go

@@ -17,6 +17,7 @@ package clientv3
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/url"
 	"strings"
 	"sync"
@@ -237,10 +238,14 @@ func (b *simpleBalancer) updateUnhealthy() {
 	for {
 		select {
 		case <-time.After(b.healthCheckTimeout):
+			fmt.Println(time.Now().String()[11:29], "cleanupUnhealthy 1", b.healthCheckTimeout)
 			b.cleanupUnhealthy()
+			fmt.Println(time.Now().String()[11:29], "cleanupUnhealthy 2", b.healthCheckTimeout)
 			if b.isUnhealthy(b.pinned()) {
+				fmt.Println(time.Now().String()[11:29], "b.isUnhealthy(b.pinned()) 1")
 				select {
 				case b.updateAddrsC <- notifyNext:
+					fmt.Println(time.Now().String()[11:29], "b.isUnhealthy(b.pinned()) 2")
 				case <-b.stopc:
 					return
 				}

clientv3/integration/kv_test.go

@@ -17,6 +17,7 @@ package integration
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"math/rand"
 	"os"
 	"reflect"
@@ -31,6 +32,7 @@ import (
 	"github.com/coreos/etcd/pkg/testutil"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/grpclog"
 )
 
 func TestKVPutError(t *testing.T) {
@@ -857,6 +859,7 @@ func TestKVGetOneEndpointDown(t *testing.T) {
 // TestKVGetResetLoneEndpoint ensures that if an endpoint resets and all other
 // endpoints are down, then it will reconnect.
 func TestKVGetResetLoneEndpoint(t *testing.T) {
+	fmt.Println("TestKVGetResetLoneEndpoint")
 	defer testutil.AfterTest(t)
 	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2, SkipCreatingClient: true})
 	defer clus.Terminate(t)
@@ -868,15 +871,18 @@ func TestKVGetResetLoneEndpoint(t *testing.T) {
 	}
 
 	cfg := clientv3.Config{Endpoints: eps, DialTimeout: 500 * time.Millisecond}
+	fmt.Println("clientv3.New:", eps)
+	clientv3.SetLogger(grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 4))
 	cli, err := clientv3.New(cfg)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer cli.Close()
 
-	// disconnect everything
+	fmt.Println(time.Now().String()[11:29], "disconnect everything 1:", eps)
 	clus.Members[0].Stop(t)
 	clus.Members[1].Stop(t)
+	fmt.Println(time.Now().String()[11:29], "disconnect everything 2:", eps)
 
 	// have Get try to reconnect
 	donec := make(chan struct{})
@@ -886,18 +892,24 @@ func TestKVGetResetLoneEndpoint(t *testing.T) {
 		// takes >5-second to unpin and repin an endpoint
 		// TODO: decrease timeout when balancer switch rewrite
 		ctx, cancel := context.WithTimeout(context.TODO(), 7*time.Second)
-		if _, err := cli.Get(ctx, "abc", clientv3.WithSerializable()); err != nil {
-			t.Fatal(err)
+		fmt.Println(time.Now().String()[11:29], "get WithSerializable 1")
+		_, err := cli.Get(ctx, "abc", clientv3.WithSerializable())
+		fmt.Println(time.Now().String()[11:29], "get WithSerializable 2", err)
+		if err != nil {
+			panic(err)
 		}
 		cancel()
 		close(donec)
 	}()
 	time.Sleep(500 * time.Millisecond)
+	fmt.Println(time.Now().String()[11:29], "Restart 1:", eps[0])
 	clus.Members[0].Restart(t)
+	fmt.Println(time.Now().String()[11:29], "Restart 2:", eps[0])
 	select {
 	case <-time.After(10 * time.Second):
-		t.Fatalf("timed out waiting for Get")
+		panic("timed out waiting for Get")
 	case <-donec:
+		fmt.Println("DONE")
 	}
 }
 

clientv3/integration/network_partition_test.go

@@ -211,3 +211,48 @@ func testBalancerUnderNetworkPartitionWatch(t *testing.T, isolateLeader bool) {
 		t.Fatal("took too long to detect leader lost")
 	}
 }
+
+// TestBalancerUnderNetworkPartitionLinearizableGetLeaderElection ensures balancer
+// switches endpoint when leader fails and linearizable get requests returns
+// "etcdserver: request timed out".
+func TestBalancerUnderNetworkPartitionLinearizableGetLeaderElection(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{
+		Size:               3,
+		SkipCreatingClient: true,
+	})
+	defer clus.Terminate(t)
+	eps := []string{clus.Members[0].GRPCAddr(), clus.Members[1].GRPCAddr(), clus.Members[2].GRPCAddr()}
+
+	lead := clus.WaitLeader(t)
+
+	timeout := 3 * clus.Members[(lead+1)%2].ServerConfig.ReqTimeout()
+
+	cli, err := clientv3.New(clientv3.Config{
+		Endpoints:   []string{eps[(lead+1)%2]},
+		DialTimeout: 1 * time.Second,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cli.Close()
+
+	// wait for non-leader to be pinned
+	mustWaitPinReady(t, cli)
+
+	// add all eps to list, so that when the original pined one fails
+	// the client can switch to other available eps
+	cli.SetEndpoints(eps[lead], eps[(lead+1)%2])
+
+	// isolate leader
+	clus.Members[lead].InjectPartition(t, clus.Members[(lead+1)%3], clus.Members[(lead+2)%3])
+
+	// expects balancer endpoint switch while ongoing leader election
+	ctx, cancel := context.WithTimeout(context.TODO(), timeout)
+	_, err = cli.Get(ctx, "a")
+	cancel()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

test

@@ -101,11 +101,11 @@ function integration_pass {
 }
 
 function integration_extra {
-	go test -timeout 1m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/client/integration"
+	# go test -timeout 1m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/client/integration"
 	go test -timeout 15m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration"
-	go test -timeout 1m -v -cpu 1,2,4 "$@" "${REPO_PATH}/contrib/raftexample"
-	go test -timeout 5m -v ${RACE} -tags v2v3 "$@" "${REPO_PATH}/store"
-	go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example "$@" "${TEST[@]}"
+	# go test -timeout 1m -v -cpu 1,2,4 "$@" "${REPO_PATH}/contrib/raftexample"
+	# go test -timeout 5m -v ${RACE} -tags v2v3 "$@" "${REPO_PATH}/store"
+	# go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example "$@" "${TEST[@]}"
 }
 
 function functional_pass {
@@ -219,17 +219,17 @@ function cov_pass {
 
 function e2e_pass {
 	echo "Running e2e tests..."
-	go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e"
+	# go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e"
 }
 
 function integration_e2e_pass {
 	echo "Running integration and e2e tests..."
 
-	go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" &
-	e2epid="$!"
+	# go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" &
+	# e2epid="$!"
 	go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" &
 	intpid="$!"
-	wait $e2epid
+	# wait $e2epid
 	wait $intpid
 	integration_extra "$@"
 }