From 7d3b238af8e58bc4862750ad57e3ce0f86e00ddb Mon Sep 17 00:00:00 2001 From: xUser5000 Date: Thu, 9 Oct 2025 02:00:01 +0300 Subject: [PATCH 1/4] test: add promote with auth e2e tests Adds two test cases for the member promote operation when auth is enabled. In the first case, the member promote request is submitted to the leader, and in the second case it's submitted to the follower. The leader case succeeds. In the other case, the follower forwards the promote request to the leader. But, the forwarded request fails due to a bug. The logs on the leader include this message: ``` failed to promote a member ``` as well as the error: ``` auth: user name is empty ``` Signed-off-by: xUser5000 --- tests/e2e/ctl_v3_member_test.go | 65 +++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/e2e/ctl_v3_member_test.go b/tests/e2e/ctl_v3_member_test.go index e9d302716d56..2893e572c3dd 100644 --- a/tests/e2e/ctl_v3_member_test.go +++ b/tests/e2e/ctl_v3_member_test.go @@ -15,6 +15,7 @@ package e2e import ( + "context" "encoding/json" "errors" "fmt" @@ -46,6 +47,15 @@ func TestCtlV3MemberAdd(t *testing.T) { testCtl(t, memberAddTest) } func TestCtlV3MemberAddAsLearner(t *testing.T) { testCtl(t, memberAddAsLearnerTest) } func TestCtlV3MemberUpdate(t *testing.T) { testCtl(t, memberUpdateTest) } + +func TestCtlV3MemberPromoteWithAuthFromLeader(t *testing.T) { + testCtl(t, memberPromoteWithAuth(false), withTestTimeout(30*time.Second)) +} + +func TestCtlV3MemberPromoteWithAuthFromFollower(t *testing.T) { + testCtl(t, memberPromoteWithAuth(true), withTestTimeout(30*time.Second)) +} + func TestCtlV3MemberUpdateNoTLS(t *testing.T) { testCtl(t, memberUpdateTest, withCfg(*e2e.NewConfigNoTLS())) } @@ -263,6 +273,61 @@ func memberAddAsLearnerTest(cx ctlCtx) { require.NoError(cx.t, ctlV3MemberAdd(cx, peerURL, true)) } +func memberPromoteWithAuth(fromFollower bool) func(cx ctlCtx) { + return func(cx ctlCtx) { + ctx := context.Background() + + require.NoError(cx.t, authEnable(cx)) + cx.user, cx.pass = "root", "root" + + // Add a regular member + _, err := cx.epc.StartNewProc(ctx, nil, cx.t, false, e2e.WithAuth("root", "root")) + require.NoError(cx.t, err) + + var learnerID uint64 + var addErr error + for { + // Add a learner once the cluster is healthy + learnerID, addErr = cx.epc.StartNewProc(ctx, nil, cx.t, true, e2e.WithAuth("root", "root")) + if addErr != nil { + if strings.Contains(addErr.Error(), "etcdserver: unhealthy cluster") { + time.Sleep(1 * time.Second) + continue + } + } + break + } + require.NoError(cx.t, addErr) + + var leaderIdx int + var followerIdx int + + // Determine the index of the leader and the follower + for idx, proc := range cx.epc.Procs[:2] { + status, err := proc.Etcdctl(e2e.WithAuth("root", "root")).Status(ctx) + require.NoError(cx.t, err) + + if status[0].Header.MemberId == status[0].Leader { + leaderIdx = idx + } else { + followerIdx = idx + } + } + + if fromFollower { + _, err = cx.epc.Procs[followerIdx]. + Etcdctl(e2e.WithAuth("root", "root")). + MemberPromote(ctx, learnerID) + } else { + _, err = cx.epc.Procs[leaderIdx]. + Etcdctl(e2e.WithAuth("root", "root")). + MemberPromote(ctx, learnerID) + } + + require.NoError(cx.t, err) + } +} + func ctlV3MemberAdd(cx ctlCtx, peerURL string, isLearner bool) error { cmdArgs := append(cx.PrefixArgs(), "member", "add", "newmember", fmt.Sprintf("--peer-urls=%s", peerURL)) asLearner := " " From 88717004fb70515b81862433b09e0c087d8f7c3f Mon Sep 17 00:00:00 2001 From: xUser5000 Date: Sun, 12 Oct 2025 23:20:16 +0300 Subject: [PATCH 2/4] etcdserver: fix cannot promote with auth from follower When auth is enabled, sending a promotion request to a follower node was failing because the auth token was not being propagated when the follower forwards the request to the leader. Signed-off-by: xUser5000 --- server/etcdserver/api/etcdhttp/peer.go | 11 ++++++++++- server/etcdserver/cluster_util.go | 10 ++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/server/etcdserver/api/etcdhttp/peer.go b/server/etcdserver/api/etcdhttp/peer.go index de5948d30f50..2df6c0d88d5e 100644 --- a/server/etcdserver/api/etcdhttp/peer.go +++ b/server/etcdserver/api/etcdhttp/peer.go @@ -23,7 +23,9 @@ import ( "strings" "go.uber.org/zap" + "google.golang.org/grpc/metadata" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api" @@ -137,7 +139,14 @@ func (h *peerMemberPromoteHandler) ServeHTTP(w http.ResponseWriter, r *http.Requ return } - resp, err := h.server.PromoteMember(r.Context(), id) + // reconstruct gRPC metadata from HTTP header (if present) so admin check can pass + ctx := r.Context() + if tok := r.Header.Get("Authorization"); tok != "" { + md := metadata.New(map[string]string{rpctypes.TokenFieldNameGRPC: tok}) + ctx = metadata.NewIncomingContext(ctx, md) + } + + resp, err := h.server.PromoteMember(ctx, id) if err != nil { switch { case errorspkg.Is(err, membership.ErrIDNotFound): diff --git a/server/etcdserver/cluster_util.go b/server/etcdserver/cluster_util.go index 425ed971cdf8..d25c9920bbd7 100644 --- a/server/etcdserver/cluster_util.go +++ b/server/etcdserver/cluster_util.go @@ -27,7 +27,9 @@ import ( "github.com/coreos/go-semver/semver" "go.uber.org/zap" + "google.golang.org/grpc/metadata" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" @@ -305,6 +307,14 @@ func promoteMemberHTTP(ctx context.Context, url string, id uint64, peerRt http.R if err != nil { return nil, err } + + // add the auth token via HTTP header if present in gRPC metadata + if md, ok := metadata.FromIncomingContext(ctx); ok { + if ts := md.Get(rpctypes.TokenFieldNameGRPC); len(ts) > 0 { + req.Header.Set("Authorization", ts[0]) + } + } + req = req.WithContext(ctx) resp, err := cc.Do(req) if err != nil { From 179f3e8391554412a74fa5d7eb16b642832a8f21 Mon Sep 17 00:00:00 2001 From: xUser5000 Date: Wed, 29 Oct 2025 00:29:16 +0300 Subject: [PATCH 3/4] etcdserver: follow convention to extract auth token in cluster_util.go Signed-off-by: xUser5000 --- server/etcdserver/cluster_util.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/etcdserver/cluster_util.go b/server/etcdserver/cluster_util.go index d25c9920bbd7..d791907c4380 100644 --- a/server/etcdserver/cluster_util.go +++ b/server/etcdserver/cluster_util.go @@ -310,8 +310,14 @@ func promoteMemberHTTP(ctx context.Context, url string, id uint64, peerRt http.R // add the auth token via HTTP header if present in gRPC metadata if md, ok := metadata.FromIncomingContext(ctx); ok { - if ts := md.Get(rpctypes.TokenFieldNameGRPC); len(ts) > 0 { - req.Header.Set("Authorization", ts[0]) + ts, ok := md[rpctypes.TokenFieldNameGRPC] + if !ok { + ts, ok = md[rpctypes.TokenFieldNameSwagger] + } + + if ok && len(ts) > 0 { + token := ts[0] + req.Header.Set("Authorization", token) } } From 5410751ffd458f2bcb5b96f4ad46b24cbe610f4f Mon Sep 17 00:00:00 2001 From: xUser5000 Date: Wed, 29 Oct 2025 00:45:37 +0300 Subject: [PATCH 4/4] tests: use WaitLeader() in memberPromoteWithAuth() Signed-off-by: xUser5000 --- tests/e2e/ctl_v3_member_test.go | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/tests/e2e/ctl_v3_member_test.go b/tests/e2e/ctl_v3_member_test.go index 2893e572c3dd..b3a54292b377 100644 --- a/tests/e2e/ctl_v3_member_test.go +++ b/tests/e2e/ctl_v3_member_test.go @@ -277,18 +277,15 @@ func memberPromoteWithAuth(fromFollower bool) func(cx ctlCtx) { return func(cx ctlCtx) { ctx := context.Background() - require.NoError(cx.t, authEnable(cx)) - cx.user, cx.pass = "root", "root" - // Add a regular member - _, err := cx.epc.StartNewProc(ctx, nil, cx.t, false, e2e.WithAuth("root", "root")) + _, err := cx.epc.StartNewProc(ctx, nil, cx.t, false) require.NoError(cx.t, err) var learnerID uint64 var addErr error for { // Add a learner once the cluster is healthy - learnerID, addErr = cx.epc.StartNewProc(ctx, nil, cx.t, true, e2e.WithAuth("root", "root")) + learnerID, addErr = cx.epc.StartNewProc(ctx, nil, cx.t, true) if addErr != nil { if strings.Contains(addErr.Error(), "etcdserver: unhealthy cluster") { time.Sleep(1 * time.Second) @@ -299,20 +296,11 @@ func memberPromoteWithAuth(fromFollower bool) func(cx ctlCtx) { } require.NoError(cx.t, addErr) - var leaderIdx int - var followerIdx int - - // Determine the index of the leader and the follower - for idx, proc := range cx.epc.Procs[:2] { - status, err := proc.Etcdctl(e2e.WithAuth("root", "root")).Status(ctx) - require.NoError(cx.t, err) + leaderIdx := cx.epc.WaitLeader(cx.t) + followerIdx := (leaderIdx + 1) % len(cx.epc.Procs) - if status[0].Header.MemberId == status[0].Leader { - leaderIdx = idx - } else { - followerIdx = idx - } - } + require.NoError(cx.t, authEnable(cx)) + cx.user, cx.pass = "root", "root" if fromFollower { _, err = cx.epc.Procs[followerIdx].