roachtest: teach cluster-init to reproduce startup crash

Teach the cluster-init test to reproduce the crash during bootup
described in issue #25771. The trick is to send requests with a session
cookie, as the admin UI would, while the cluster is waiting for init.

Release note: None

pkg/cmd/roachtest/cluster_init.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/server"
+	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"golang.org/x/sync/errgroup"
 )
@@ -143,16 +145,37 @@ func runClusterInit(ctx context.Context, t *test, c *cluster) {
 				{"/_status/nodes", http.StatusNotFound},
 			}
 			for _, tc := range httpTests {
-				resp, err := http.Get(urlMap[1] + tc.endpoint)
-				if err != nil {
-					t.Fatalf("unexpected error hitting %s endpoint: %v", tc.endpoint, err)
-				}
-				defer resp.Body.Close()
-				if resp.StatusCode != tc.expectedStatus {
-					bodyBytes, _ := ioutil.ReadAll(resp.Body)
-					t.Fatalf("unexpected response code %d (expected %d) hitting %s endpoint: %v",
-						resp.StatusCode, tc.expectedStatus, tc.endpoint, string(bodyBytes))
+				for _, withCookie := range []bool{false, true} {
+					req, err := http.NewRequest("GET", urlMap[1]+tc.endpoint, nil /* body */)
+					if err != nil {
+						t.Fatalf("unexpected error while constructing request for %s: %s", tc.endpoint, err)
+					}
+					if withCookie {
+						// Prevent regression of #25771 by also sending authenticated
+						// requests, like would be sent if an admin UI were open against
+						// this node while it booted.
+						cookie, err := server.EncodeSessionCookie(&serverpb.SessionCookie{
+							// The actual contents of the cookie don't matter; the presence of
+							// a valid encoded cookie is enough to trigger the authentication
+							// code paths.
+						})
+						if err != nil {
+							t.Fatal(err)
+						}
+						req.AddCookie(cookie)
+					}
+					resp, err := http.DefaultClient.Do(req)
+					if err != nil {
+						t.Fatalf("unexpected error hitting %s endpoint: %v", tc.endpoint, err)
+					}
+					defer resp.Body.Close()
+					if resp.StatusCode != tc.expectedStatus {
+						bodyBytes, _ := ioutil.ReadAll(resp.Body)
+						t.Fatalf("unexpected response code %d (expected %d) hitting %s endpoint: %v",
+							resp.StatusCode, tc.expectedStatus, tc.endpoint, string(bodyBytes))
+					}
 				}
+
 			}
 
 			c.Run(ctx, c.Node(initNode),

pkg/server/authentication.go

@@ -139,7 +139,7 @@ func (s *authenticationServer) UserLogin(
 		ID:     id,
 		Secret: secret,
 	}
-	cookie, err := encodeSessionCookie(cookieValue)
+	cookie, err := EncodeSessionCookie(cookieValue)
 	if err != nil {
 		return nil, apiInternalError(ctx, err)
 	}
@@ -369,7 +369,7 @@ func (am *authenticationMux) ServeHTTP(w http.ResponseWriter, req *http.Request)
 	am.inner.ServeHTTP(w, req)
 }
 
-func encodeSessionCookie(sessionCookie *serverpb.SessionCookie) (*http.Cookie, error) {
+func EncodeSessionCookie(sessionCookie *serverpb.SessionCookie) (*http.Cookie, error) {
 	cookieValueBytes, err := protoutil.Marshal(sessionCookie)
 	if err != nil {
 		return nil, errors.Wrap(err, "session cookie could not be encoded")

pkg/server/authentication_test.go

@@ -545,7 +545,7 @@ func TestLogout(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	encodedCookie, err := encodeSessionCookie(cookie)
+	encodedCookie, err := EncodeSessionCookie(cookie)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/server/testserver.go

@@ -569,7 +569,7 @@ func (ts *TestServer) getAuthenticatedHTTPClientAndCookie() (
 				Secret: secret,
 			}
 			// Encode a session cookie and store it in a cookie jar.
-			cookie, err := encodeSessionCookie(rawCookie)
+			cookie, err := EncodeSessionCookie(rawCookie)
 			if err != nil {
 				return err
 			}