From 1661e30a52851239042429d6ce746f486f88783e Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Fri, 2 Feb 2024 15:13:21 -0800 Subject: [PATCH 01/21] Use dios gate on readfull Signed-off-by: Waldemar Quevedo --- server/filestore.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/filestore.go b/server/filestore.go index 861fe4dc9f3..24e2242a7f9 100644 --- a/server/filestore.go +++ b/server/filestore.go @@ -5553,7 +5553,9 @@ func (mb *msgBlock) loadBlock(buf []byte) ([]byte, error) { buf = buf[:sz] } + <-dios n, err := io.ReadFull(f, buf) + dios <- struct{}{} // On success capture raw bytes size. if err == nil { mb.rbytes = uint64(n) From e6f28e545ad1edde85937c67b1a59be324d81456 Mon Sep 17 00:00:00 2001 From: aricart Date: Mon, 5 Feb 2024 11:38:31 -0400 Subject: [PATCH 02/21] [FIX] [authcallout] when using authcallout with operator mode, the auth violation error was not sent to the client, thus the client would simply timeout. --- server/auth_callout.go | 6 ++ server/auth_callout_test.go | 116 +++++++++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/server/auth_callout.go b/server/auth_callout.go index 6620a4e5125..318114ebae6 100644 --- a/server/auth_callout.go +++ b/server/auth_callout.go @@ -241,30 +241,35 @@ func (s *Server) processClientOrLeafCallout(c *client, opts *Options) (authorize arc, err := decodeResponse(rc, rmsg, racc) if err != nil { + c.authViolation() respCh <- titleCase(err.Error()) return } vr := jwt.CreateValidationResults() arc.Validate(vr) if len(vr.Issues) > 0 { + c.authViolation() respCh <- fmt.Sprintf("Error validating user JWT: %v", vr.Issues[0]) return } // Make sure that the user is what we requested. if arc.Subject != pub { + c.authViolation() respCh <- fmt.Sprintf("Expected authorized user of %q but got %q on account %q", pub, arc.Subject, racc.Name) return } expiration, allowedConnTypes, err := getExpirationAndAllowedConnections(arc, racc.Name) if err != nil { + c.authViolation() respCh <- titleCase(err.Error()) return } targetAcc, err := assignAccountAndPermissions(arc, racc.Name) if err != nil { + c.authViolation() respCh <- titleCase(err.Error()) return } @@ -280,6 +285,7 @@ func (s *Server) processClientOrLeafCallout(c *client, opts *Options) (authorize // Build internal user and bind to the targeted account. nkuser := buildInternalNkeyUser(arc, allowedConnTypes, targetAcc) if err := c.RegisterNkeyUser(nkuser); err != nil { + c.authViolation() respCh <- fmt.Sprintf("Could not register auth callout user: %v", err) return } diff --git a/server/auth_callout_test.go b/server/auth_callout_test.go index 292da407799..d8634d60cba 100644 --- a/server/auth_callout_test.go +++ b/server/auth_callout_test.go @@ -820,7 +820,6 @@ func testAuthCalloutScopedUser(t *testing.T, allowAnyAccount bool) { // Send the signing key token. This should switch us to the test account, but the user // is signed with the account signing key nc := ac.Connect(nats.UserCredentials(creds), nats.Token(scopedToken)) - require_NoError(t, err) resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) @@ -1713,3 +1712,118 @@ func TestAuthCalloutWSClientTLSCerts(t *testing.T) { require_Equal(t, userInfo.UserID, "dlc") require_Equal(t, userInfo.Account, "FOO") } + +func testConfClientClose(t *testing.T, respondNil bool) { + conf := ` + listen: "127.0.0.1:-1" + server_name: ZZ + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + } + authorization { + timeout: 1s + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + if respondNil { + m.Respond(nil) + } else { + m.Respond(serviceResponse(t, user, si.ID, "", "not today", 0)) + } + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + // This one will use callout since not defined in server config. + _, err := at.NewClient(nats.UserInfo("a", "x")) + require_Error(t, err) + require_True(t, strings.Contains(strings.ToLower(err.Error()), nats.AUTHORIZATION_ERR)) +} + +func TestAuthCallout_ClientAuthErrorConf(t *testing.T) { + testConfClientClose(t, true) + testConfClientClose(t, false) +} + +func testAuthCall_ClientAuthErrorOperatorMode(t *testing.T, respondNil bool) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH service account. + akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + + apub, err := akp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, creds := createAuthServiceUser(t, akp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(apub) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add("*") + + // the scope for the bearer token which has no permissions + sentinelScope, authKP := newScopedRole(t, "sentinel", nil, nil, false) + sentinelScope.Template.Sub.Deny.Add(">") + sentinelScope.Template.Pub.Deny.Add(">") + sentinelScope.Template.Limits.Subs = 0 + sentinelScope.Template.Payload = 0 + authClaim.SigningKeys.AddScopedSigner(sentinelScope) + + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + conf := fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + } + `, ojwt, spub, apub, authJwt, spub, sysJwt) + + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + if respondNil { + m.Respond(nil) + } else { + m.Respond(serviceResponse(t, user, si.ID, "", "not today", 0)) + } + } + + ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer ac.Cleanup() + + // Bearer token - this has no permissions see sentinelScope + // This is used by all users, and the customization will be in other connect args. + // This needs to also be bound to the authorization account. + creds = createScopedUser(t, akp, authKP) + defer removeFile(t, creds) + + // Send the signing key token. This should switch us to the test account, but the user + // is signed with the account signing key + _, err = ac.NewClient(nats.UserCredentials(creds)) + require_Error(t, err) + require_True(t, strings.Contains(strings.ToLower(err.Error()), nats.AUTHORIZATION_ERR)) +} + +func TestAuthCallout_ClientAuthErrorOperatorMode(t *testing.T) { + testAuthCall_ClientAuthErrorOperatorMode(t, true) + testAuthCall_ClientAuthErrorOperatorMode(t, false) +} From e2fc85961dc66a30a8accd7d78de862b8106a630 Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Sat, 3 Feb 2024 13:52:01 -0800 Subject: [PATCH 03/21] test: update certs for TestTLSClientCertificateCNBasedAuth Signed-off-by: Waldemar Quevedo --- test/configs/certs/tlsauth/ca.pem | 36 +++++++------- test/configs/certs/tlsauth/client-key.pem | 55 +++++++++++----------- test/configs/certs/tlsauth/client.pem | 39 ++++++++------- test/configs/certs/tlsauth/client2-key.pem | 55 +++++++++++----------- test/configs/certs/tlsauth/client2.pem | 39 ++++++++------- test/configs/certs/tlsauth/server-key.pem | 55 +++++++++++----------- test/configs/certs/tlsauth/server.pem | 37 +++++++-------- 7 files changed, 156 insertions(+), 160 deletions(-) diff --git a/test/configs/certs/tlsauth/ca.pem b/test/configs/certs/tlsauth/ca.pem index 33553027008..e30e6999808 100644 --- a/test/configs/certs/tlsauth/ca.pem +++ b/test/configs/certs/tlsauth/ca.pem @@ -1,21 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDaDCCAlCgAwIBAgIUWyR/qbLooFMu+VcvmQhLAjokntQwDQYJKoZIhvcNAQEL -BQAwTDEkMCIGA1UEChMbU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuMRAwDgYD -VQQLEwdOQVRTLmlvMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTkwMjA0MTk1MDAw -WhcNMjQwMjAzMTk1MDAwWjBMMSQwIgYDVQQKExtTeW5hZGlhIENvbW11bmljYXRp -b25zIEluYy4xEDAOBgNVBAsTB05BVFMuaW8xEjAQBgNVBAMTCWxvY2FsaG9zdDCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN9ryA3PTdAPjC2VQkjy9JXJ -bOq2GpvGU+2/gC3TNRXOPJ5ZVy4svV8C9VA9t8gIbQHTYMzBFxyGz0+a/9+DEXot -crcVvsqaE5mewU9yjifDqUCGqOn9fo/zsYwD96KYtukEZ73D1Pyv+7EmkHNYqBKB -4/1gY/7AuuBcNp5bSpC4isGySZlL0wDjURyjfInrbDdMZi3QK2lPZP1okLZG5SCX -7pQM9riHwnzN94HINTzLTUdjxDBrm0Av9HCEeGT+iXwtXIhNaTkxjEy3a6b2saVl -wcaqcZbdGmJVgoncNlA3+277BPOAfbw4X5nGATaWPWxStkqeuhSaxahbCLNJGJcC -AwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0O -BBYEFG8G2+G/R8ovyXZoCjtIco9u9hrLMA0GCSqGSIb3DQEBCwUAA4IBAQBmuKij -sa+RKEoSVrdUWYwAhQJd17I1crhyLjzk3c5k4cXSIUM0XlGK81GZdPRV5EVym7FN -n8rhjAYizFykFbIcmiUrNa73jm2QTdMiL8WEzywNB0/X+XSJd+I1VeWOvYJMPTiY -KH/vcNYugVeWUzn6EF+iWnlpS9IHxcDvm6yjMJ242+KQWO7DGkHzbadB/BcryAdz -v6oBlHTJoPqgHUwaHfnTfqCQPTaTACUSFGNEnLuuXvLbbhZlpmLHRoqBiwpa0YQW -1EAICjLa6q5vSDSBrYJL2tIZz2vv/powIWMU1tdGFSALtpMucUH5Opi0Eaa+3cQB -fvl1Mck/CPY8e4/j +MIIDBTCCAe2gAwIBAgIUYm7HlYXiXY8u67v2vKjU5LUCHDUwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MjRaFw0zNDAxMzEy +MTQ4MjRaMBIxEDAOBgNVBAMMB05BVFMgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9GEoJ84UWZTsnovuz3StEtri3NdKlOqzNCGvurd3rZgop/jZB +RtJXQz9ZxdKx1ARpDV1m3CrQe63UbBRXJxA2XGfmBQ/BPo3IPEXJOsNEs9x5RsSL +RiJ2re7jbWKeQv/ucQPdmLJumAp+TGAzdOM/AnDaVTSLPoARp/Va8Frs7iFfPpuJ +tObvux4qnb/hxS2z39MWjyeM0dVOmjGwx9opxcE0hNI5ZutkoNxpmRayZqJSe85V +BSPGsuBwgncvA2GWTNIGFfN2oxQhSuI8yM7+l/0+BHFWfm2G7/09tWDvFnWTSpTQ +VISM3+6Wh91c6qSd0wsIb8q6jADAD6H8yhT/AgMBAAGjUzBRMB0GA1UdDgQWBBQk +vMZGyNfVHU3oTUASSfYiT4arNTAfBgNVHSMEGDAWgBQkvMZGyNfVHU3oTUASSfYi +T4arNTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBxN8yaacok +1lyg7eTEv7/Sn4VGkfbuWm9ZQ3DPLKJhbe7uVxNChRrR8nKshwlGv1Wa8i0c0lJ3 +O+Uw24gjfhl/8zhFyxh4ijllj6/FVBNqsTvnGQqtKJP4h8kScUIH21mQ6JQAvh4e +RY1sjPwZp+6vvogSrgQQ32jBaa8vfzcL2wECvnT1YqePVZYuRqEBjIvyG0ALlmE9 +DqZ8gH+W8E5IVulLVJxnYArCT1dW5AyM2fBETLB3PAWvSBkaCBl6QR+hLuyeR4vT +m6Qx9EKr8MgIpiH7psnx8C9eF5j5HiwHfhwAdWD9W2tRzTxSZP2LJ9E+qjaohdLf +6NYxXL8AHa17 -----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/client-key.pem b/test/configs/certs/tlsauth/client-key.pem index 002d091fc9c..1727e457db7 100644 --- a/test/configs/certs/tlsauth/client-key.pem +++ b/test/configs/certs/tlsauth/client-key.pem @@ -1,27 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEAr5QLVY0FeeEO+mPQDSGZSael1cO0cfABww7r5fNkxFRipJL0 -4uOKsP359dGo/Ay8MWS2mlAooX1jRA28YWghNjgZwv05bbmcQ2cGo9/7TTWKxusk -zHYgYLc9WNbgJnSGNceykQxIgnI4S/pF+fxpoJbxTOu71tviky5SAP93IU8JpfB4 -aIpSo0KfA0wDpauYD1orjrXY7JMvL/NRLLYvkTvJaMeuEAWuW3UacCV1wfT91oQW -FBaFupsatNT0UPJ+eMrfG0Gg6cM9bfU+ZNZROQOOR0gAM9XD4knuOYQsvdKWrEku -N8bBqCPOhCIYYCtUan607phtQoIBuwyxrEkSJQIDAQABAoIBAFn+IJ0V7gOdVmcC -d+XzHbWB518cs0VfBhgrcr/nM/PpaLH/3OLaTAER/GeBsgKWqHMMswd/JIQ5V4LP -I4otrDA1KwclcaUK6MwnZ2DhcdYOJnZ0meTuewP3h8scP8GWIiA4ng74Y8Xws2hF -/E34kU9NbprFjP7Ar25O5Js8VZxNJBWZXZqd4znTj8d4eqbZghM12RaUNQ1YSgB9 -dC9a4siubAWw3mZXX9T8te/uWbo7gY52vaMlO1nV+waZzkqdF7xizAYzYW+woAu5 -lMRnUylRFKtuAUZGsDATElVKeirWT/kkAjslRzLw0daOYfe+c321bE9fFwFi5b3e -DHSwpbUCgYEAyydwejT59DHvrQ1HRnKCL4+rvrCtFlHoJiSOKX8F38tVjCwFstHO -on+AxJMU8Om528aJ2RDPDzy1Cqn+cRAmgIsvAwa2JyBGqkN9nXq4Ji8eBL42ZNzK -wbnnFfm+qkBNXXon1gSnMN+8w9zX1dmIsoGmunZ/Ew0EB1bDsBEcI1cCgYEA3UBD -QkvFu6SlYq8ts5RcIn5bGgPkvudbRTb2IDYVMtLmPdDQP1X6s4NFdHErqvfwyIez -2WQLjLjUfU8fylEKxZL9cdHF/FAa5brnr3DAVT1Gsthv8WFFUkuZ52uAte4mGTL0 -FnGaz6mqoIrM/uNGKAgozMZPrDFobqsh5maepOMCgYAO3rMn7srA6grOEuO9r1IC -IzUB/zKcKKCichiJxwdqCxsW6H3+SccjM8v8F3v36lO1V4HthoJxbhMeVbUPF4yJ -6iYlxY79rCof+lKufTYPbXF4DWgz18lrhqz4edBP6+b9yZwy2SJXvHi3qWmO+J49 -2qmWimfgwBokY2BtecMifwKBgHiiMknycISIGBi/dQamDLpN9LQxjUY9dPk/J2GW -u2YzsY/gy7rM0V2RZIxBrFKSz3k27GvKbbWzjUAppSa1m07wfznQ68dPkerSRsLU -kjmnqGWZNygAJkDhsa+JYOtRRvqUWpvmI0e4tazFIVKUbssi78P/GK/FXLCCpIAw -Ua2LAoGABl1qtut/+NyOXZHJVl8tZIzeJOWkuMoKf0dOWmOsQl0Si2tGeB/ExI+O -mnhqozoV3nBGE8AyzMnUh/C6/tH5be3w/y3pTJ4rayffnYsYgf7mySAopjIbpW7w -iORnunwB7qnzx8yIS6rK0kpyvfp0P+bIOTwT1nEbw9wnBSjmCO4= ------END RSA PRIVATE KEY----- \ No newline at end of file +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjAaejJ5wqwsst +7T71LQCdsbRpctZU2ZbTIOalEvHNikkdB2gIqxlmswxITFVtNp2IvVhcd4KaNGdu +a4uUc7BifG6XwFtmw54SPTlbFtZbr1y4mjF6+Wm3tV0XnMtM18wvkRm7wbYG5fz8 +BfWl4tbCDF8lnuC3YHFwVJst+o1HoZ4YNutDFJqMXkHuxEPsI5PmS3XHHpttPQGr +vKnoYR0y0Cf4y3LCfQCQ+bFe2JS335WmJy2KVm+dpN+L2ASO8VvV/YTltSy8f4HQ +3Dd/VjZtri7ANP61Xrrx4OJS6IRkPo3uxaR306oKyF9a4irX1KY+Vi1gpYOX9foc +mVumCWCrAgMBAAECggEARk+F7CG/QlCQjEhb0ixtqheHPr7KhYHvhTUZV5XC2Aow +fEWAEdEfnUVY5GyMopWewOcPUJ86JeK5xI69/7QhHnIWz/0oT7zMF4jyDwDcSGLt +RzE3a5heid/Aflli9cvVZqUbaPnm1rXoeBrn+PxN7xigB92uh1qhw7ay0tPSkdUL +sIA2fkouwAYIE5TDhKk4lF8Xg6KlYJEesFDGTVsaGL8/RH7VEH/Z7EssiHF60m4l +Oj6Vg7gxMB1UWQFw2/LDMrDw0bLVdAoOC0M7NLXlFdoeKPREvFpIqw8BWMF3NxRV +OpUiLB/tv6A7nqevaa8KoWGUlCVY2DkCkgCmg+K8pQKBgQDVtb3nEJrOR496BaLn +pHKV+pEQjlVfcy36CoTa2PblbjkgSyA6wL9Itpkw4OF/Q8dop0qlbqZM9gUdjEhB +6DfVINNMGx4SkWDtBvPwouCRIWRvZUXdVuZMgDpUY+5n2nEkUHbmSUEtPbhHn5dp +9Tn1dsjNjweDTx2Q6nDuKUlSVwKBgQDDQ1fyyp6qI6opeFkfOsZ78Yk2lM6tRSkI +JqwdOwZoSptZfiaoAQtmwU3zxioFgIv4FjbPCosJyej0Egv7oyMucZHPGhP6kZDJ +LxZ0Jp0kDYYFCmUjOEpDf0mSmFEwmyPDb8052To3EuA4anEQd1KURG/0yKE4efwP +0m3ml9R3zQKBgAh1cxjMPXRgvLsVsgb9KVPqYQeIurRWeMFm3S9UWyFlpXkzwAjT +TD7yi0m1/Pbuldv8kyXNJWPycO1keg+xw1P6QqLGiAAwJOf82Hbz23OjILiQB53l +LKRmhuiENBGEQeowDSS8TYoe4UZkeLfG7w5aL0SDnsaBwSfVP7cNh0ttAoGAEJDO +DVMTUuvjq9EB/pxF6o37Th4hyqFrcb2WLIStbnul4lnJfcdY6EbODjhpqD3XohyA +WeBTG2l90fcV/StB+Na5wBA+Uau31Nmh1gjQnBZpoFPZcLt90WwjGcTCXpVK23HI +v3emcLWxQBgHr5Xv85Q6y1GaG+h9cfowSLfo1qECgYEAmqGyMa9OZ44+zni3nq2A +JM2t4B5QJdG36R/2m4J7odUnmsaRVxw6JxaoF0CctFxGs/zdumwuhYeTfq6Lj/ly +N1kJbvJJpguN1h4sOOLEk646mZzK2h5bMxsmPU2kNZFf1EUlB+Ro8i1TDvGY26Qs +qgjd6RlSZFBmSfVS47ihEQA= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/client.pem b/test/configs/certs/tlsauth/client.pem index 9b70e353cb1..8c873e0b02b 100644 --- a/test/configs/certs/tlsauth/client.pem +++ b/test/configs/certs/tlsauth/client.pem @@ -1,21 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDhjCCAm6gAwIBAgIUfLE3jBpwGplOUDcpbxIsvRInDGQwDQYJKoZIhvcNAQEL -BQAwTDEkMCIGA1UEChMbU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuMRAwDgYD -VQQLEwdOQVRTLmlvMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTkwMjA0MTk1MTAw -WhcNMjQwMjAzMTk1MTAwWjAoMRAwDgYDVQQLEwdOQVRTLmlvMRQwEgYDVQQDEwtl -eGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK+UC1WN -BXnhDvpj0A0hmUmnpdXDtHHwAcMO6+XzZMRUYqSS9OLjirD9+fXRqPwMvDFktppQ -KKF9Y0QNvGFoITY4GcL9OW25nENnBqPf+001isbrJMx2IGC3PVjW4CZ0hjXHspEM -SIJyOEv6Rfn8aaCW8Uzru9bb4pMuUgD/dyFPCaXweGiKUqNCnwNMA6WrmA9aK461 -2OyTLy/zUSy2L5E7yWjHrhAFrlt1GnAldcH0/daEFhQWhbqbGrTU9FDyfnjK3xtB -oOnDPW31PmTWUTkDjkdIADPVw+JJ7jmELL3SlqxJLjfGwagjzoQiGGArVGp+tO6Y -bUKCAbsMsaxJEiUCAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAww -CgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUa1+2h/uPRLo1o6Ee -qOTA1hKmUlcwHwYDVR0jBBgwFoAUbwbb4b9Hyi/JdmgKO0hyj272GsswCwYDVR0R -BAQwAoIAMA0GCSqGSIb3DQEBCwUAA4IBAQBr50msBYEhIMj6b8QoBAbvdBhE9TeF -76k4PZpGVF6EoGVdyTXvO0LBNT7BFysIVgy51Bv5kyz7z9B6iOwnnTDhZIj0kGMA -KPkgyMHHRlL9iOXC/fCuf9dfBuq7u2oykILCI8VY6yzTvHtIWg0/wk6/2e13WmtU -nkI9ySxJGzaZJaVjV7UWkzzR1anwJ6Q0IaRohIByWcE43uIzNi1sA1U8cZ70C43t -JNSXp9mBodV2jLCM3bU9jpSyz8thH3ghogioG8obYSS22a+Ei4SRsmHK1B2fu6Uh -z8UJCtjq5lbVlPgZQlmIHAJaq/cK8nSccv/2KBHx5hOR8rIqwsAL6p46 ------END CERTIFICATE----- \ No newline at end of file +MIIDPjCCAiagAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipYwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ5MzRaFw0zNDAxMzEy +MTQ5MzRaMCgxEDAOBgNVBAsMB05BVFMuaW8xFDASBgNVBAMMC2V4YW1wbGUuY29t +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAowGnoyecKsLLLe0+9S0A +nbG0aXLWVNmW0yDmpRLxzYpJHQdoCKsZZrMMSExVbTadiL1YXHeCmjRnbmuLlHOw +Ynxul8BbZsOeEj05WxbWW69cuJoxevlpt7VdF5zLTNfML5EZu8G2BuX8/AX1peLW +wgxfJZ7gt2BxcFSbLfqNR6GeGDbrQxSajF5B7sRD7COT5kt1xx6bbT0Bq7yp6GEd +MtAn+Mtywn0AkPmxXtiUt9+VpictilZvnaTfi9gEjvFb1f2E5bUsvH+B0Nw3f1Y2 +ba4uwDT+tV668eDiUuiEZD6N7sWkd9OqCshfWuIq19SmPlYtYKWDl/X6HJlbpglg +qwIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93 +d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFA5GJsoT3uf5GCyL5KSFoyEuRVltMB8G +A1UdIwQYMBaAFCS8xkbI19UdTehNQBJJ9iJPhqs1MA0GCSqGSIb3DQEBCwUAA4IB +AQAmBOQOp3CR5/P27RUonsgf5MCzblt6tBPJBMiZwB8sJyl9fXCTkzdqD0rJQb7q +5s7p00rUXdeh13oRjFcoFuopMDCk3kBYnKJHXRREJNLHfp6CPUMKlg0GJUZ6v04A +V7gVuhvmynHrmlbibMwbgZtZMnRU3x8JjawEUsEhoj3O2Qfen3sNfaOBlnwVUCBQ +ygSHQ0Pto1kQS+1Pc5DCwnOZ/qh5lORPdO1MNKqeu8HiiSJfuaCrQQM9zm72CHHY +F755qy8OvWjwK0H9rCFBYSrAnYk/pTvXIeBsgNRlURS/qv1rqIEvAbXhRnw7oyvl +P4bYY4pcpk32Ir2mFQFRQnSh +-----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/client2-key.pem b/test/configs/certs/tlsauth/client2-key.pem index 9e74cd2f733..6ae5fe08955 100644 --- a/test/configs/certs/tlsauth/client2-key.pem +++ b/test/configs/certs/tlsauth/client2-key.pem @@ -1,27 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEApRvhQ78kWi/gN1DWuTrk0vhS6Zva1U1TdHvtp0lvzgYqy7Z7 -mIEkX6tVZxIV7RXrOWu+CGWtkCb83upQudYdsFqbtOSbhsmeupAsKPePYSRJuYaf -BShNOEQ+UP+xEQJOv9LCHGFz8ukcF9XXJAlo0c5e6PDLmLPbMM7NmyZlqLGkXEgZ -cZEEG+BaxfR75gYoODnxkmvpx8F9nXKpDlIRNP9A2iTIWna0AX/r7fsivvBJCwj9 -j3j9g5gmEurT5oVeCaya6N60NVSbMug4UF70NF2EAtw8TKIat73ATdgf+EFY4wO2 -xdceM9OMiurrnV3FMOHMfU148rdFbHb99zIupwIDAQABAoIBAQCRxUSj8Gzi5yP5 -EnkRPorqLG3fbEfPTJ7i18thh7ebWNyN0IXchiAcCwOypUgQcuqjXpl/hm2vOIzH -Lm6pM/4wRj70fWVGolluc31Zif/fjw88KjvZbNSIWc/+6VBmKPhn6WaRcgTRsLep -35U7bsdJfP9Uf8vw/NIHjH4Afe0A+rJQ5aX17PLvlBH44WBOGvMq2qPPWfb5PzX6 -mne+a4M0FdTjfyL3qiJW2LF8OOVVDM9p9roQfZ15eKzxzQy4EhU7jtB0Bp9oGsu+ -nh0Pwvev0VQ1PPvZoxKP8FgwHSt/wekuht4bbQT0rQRNgKuJ+H1zwVe9mFDYslLh -bnfoFwQZAoGBANVIPdzEDx3op9n4RBQ7zBSwQUG2xRg42VrbjYdFkTB52a8m553I -4XrUb6QMCqFo1Va3BpZvEXfQIHXYSssakC1b1R/FVcRqlg27wKyFdW4B0UierBjt -iUjUfto4+CZzjpmEJJJcYduLigLqC2/hzFcUFs3paDHXJJfZfmlbhtd9AoGBAMYt -nhnZzgxRfzB05zaqw/hiR+X9lDMElOYKz8GUoygLlQYzsBnoQ+YD4ZRvETL1WhgN -M2Kc3BHjPdMFNZyvoRXgrqg018KDjzCqlHH1iWXVfUgnOlTi9kX3UdEhR4HcYi8Z -EnilkBseLgpC0byOXSiPVAUt34qy++D27OlwYZ/zAoGAU2Fjtev8EPBEtqUlUFe0 -SB5D1MH0Oaz35FpS8SBUS4RHgv8Nq5S9+bwVTSfb/BA03yq8a5FOXe3C0u9VBiQD -W4g8QKhwCFK3CPVutMOUDgat39sQYspyUkOot/1vnfCtPfz4IzP0mdTqhosjH4FB -1oUnCScHsfxu9OJ1VhEPHS0CgYB+lYLIFlRDkAbC59kMFRVp4TT1lfyEfeex7LP5 -fTyeBo/gz0Eruy0rjc0X572/o/IxLLVmxrTXBCRoVoqBE7m75LELJf2u9CORPVPm -WqSxlCUa4lui/vm5hRkQkMZBD4jzdntS7sXWXHeh/D5Fx1V/49USHdQMnvi+IFsB -XNQuuwKBgE48AdaUfWoqbkR+vfMbqeTyAXOg7VfmT1m8dc8mNPTeaEjUwb4M3EaQ -fT2B+tIASHfIaqLWPqjCrLxCBz4P4+4dkTLlWN+7BTVgK4Ecie/pEg8+q3CEVV2n -Bt5/z4xQCUmj24eiZogRIZmK9OsXEy4vQk4YH09tFSpFa+WwjoVX ------END RSA PRIVATE KEY----- \ No newline at end of file +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMsHsIYdxGt8ws +eoxGQQs4K+AfvznwuW+Qbnx+eGTS+OSaLqL9u81JdXxheeAVsVjRtz5qP3X9Vu75 +gSQU+AVhFp689VNR41vaOsKvX2dzVCaWCEwv66CekAop89vyB6fm14uSr9IusixH +u6vfjuze2FZE5xcw5mSYMysqKTbM4DMjXFJXFg5mw1YC9zkh3zCWUZdmidafwOB6 +MiJkVZsAQjAwc5oaD4I/ubCdmWr8COYbbwUl32k+t0eQfj9cvd4+R7uBYVNWu1BN +g0RqNZZlIM4EVonc1J+eN7oGo57jLrwf4jCOkcg2wd6X62CuuQHhwddQek4KGthM +JUIH7Q9TAgMBAAECggEAE80u2cy9xomZUuQ4FcPNFg4IjImvTT5jMJG/sWxsNIyn +cNL6KZm1blnTQorLxs11TjRv8U9aVrvGOpTnrK+htZa+nIEPImjgRehRVS3hkCKf +6Pu8gxZEX5KHqS9SI8Ph1k8bzYD80E+kQPxC0Em/WH+NOPUyJSTkrmSk1FtQVdle +KFk0+dRm10guI/rcInPUjpY6Cv9nSiAhanRPAtAANUvwYxVEMKzx5ns/JnQfAYAK +gYTopXAT61VqW1jn4opjD+OqparjbK362q/spggHGPhSamXq8y/N9JadSx+5591u +o0oZm/UxefwP1CkE4Q6Pv9n3yiZOwVaOiLLVzmr6wQKBgQD4kC953if336Ga+BdB +Hj83qUk5U2Avp6qWLptd6z/VQXDwlL2oQaaXP83T6XfpXTMwDabUYkpo1JZrFOJl +s9eZfxIDa0eH6evphqp6s/V6EhZkI9lL59ptZlK1hv1glmFDfn7WB1Yo7d8GgUx/ +S7CKoBK2WaX8YNsW9fIunadJeQKBgQDS0EDR46KiQBrjvNK4UxdfERl2af+6Sf+J +BRlfj+YA9/zKEL3jZGWVa5rgrD8BJrTsa86QSSACsxQx5J9vPogKgogmghd9RQVd +tHvIV0C2C6zCmZIvNuzn5+wMV7bWdoABPgbuZGTpc/LCwm//1EbwdNCTom8ssE19 +NTIBb7t4KwKBgQDXOqKheAhLzkz1D1WzgSlkXSWWieeD3D8OBBVsYcPIOP4+k80V +4KML3KexkzvNynIEbg3DYcjktQ/6cP8I6Y0K0MkcRMyPl7I7Z+w+i41Hwlm5JIGI +BJ9Sk4OSw+yqsgxOkT3qvjeRAUhZLaS7pSKdJraNR1s/Ce8sFpM6YjD0oQKBgAUc +gYXVRBs0/LHq4R0Q/q8SZhCl70pgAu8ajYvwnD4HxTxM/Z2m0IO38TBjXL+1ZYuZ +7Y84BquqFeJDzc3PsVK36X8thk5GPyQPfTTVUL9ZNx4cxRuZ9FKHIAUIl2lJxD7D +dz2Od5fldMxeFIMabYHlAy2hMZrex3IyuPyp7dyzAoGAKCqmXStl7QzV6ovrjyJ1 +oMP3BcQhTe1CvOEW7pGce/IGxqB0p7gICIH4UUDEVpitgvGLkOGScsd1rS86Fn+/ +RUlQQdgRIwzrlF1USV1kJ/SUI+7BbhiXUeRUmKuzPLgLSq8K3xEW0z21juSvq64W +gtiiGjt3D3I47v7ecjJ9dIA= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/client2.pem b/test/configs/certs/tlsauth/client2.pem index 7ebc211094b..edc66519864 100644 --- a/test/configs/certs/tlsauth/client2.pem +++ b/test/configs/certs/tlsauth/client2.pem @@ -1,21 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDgzCCAmugAwIBAgIUXSH0jKq+6x2WG4RHqN8tATdptokwDQYJKoZIhvcNAQEL -BQAwTDEkMCIGA1UEChMbU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuMRAwDgYD -VQQLEwdOQVRTLmlvMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTkwMjA0MTk1ODAw -WhcNMjQwMjAzMTk1ODAwWjAlMQ0wCwYDVQQLEwRDTkNGMRQwEgYDVQQDEwtleGFt -cGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKUb4UO/JFov -4DdQ1rk65NL4Uumb2tVNU3R77adJb84GKsu2e5iBJF+rVWcSFe0V6zlrvghlrZAm -/N7qULnWHbBam7Tkm4bJnrqQLCj3j2EkSbmGnwUoTThEPlD/sRECTr/Swhxhc/Lp -HBfV1yQJaNHOXujwy5iz2zDOzZsmZaixpFxIGXGRBBvgWsX0e+YGKDg58ZJr6cfB -fZ1yqQ5SETT/QNokyFp2tAF/6+37Ir7wSQsI/Y94/YOYJhLq0+aFXgmsmujetDVU -mzLoOFBe9DRdhALcPEyiGre9wE3YH/hBWOMDtsXXHjPTjIrq651dxTDhzH1NePK3 -RWx2/fcyLqcCAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYI -KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUoS4dpE8Slaffykf+cVSc -g7IXvcYwHwYDVR0jBBgwFoAUbwbb4b9Hyi/JdmgKO0hyj272GsswCwYDVR0RBAQw -AoIAMA0GCSqGSIb3DQEBCwUAA4IBAQChjRkAiIuEXco4AkdoLO4wSN0i0b/toZ9b -U6X91UPCOQMYGLqe81DFYh3JE/+YjrwQYZz5Yb/vRVBC2HmTYkBXdP/74kRu4LCz -cdiVimz4GF2cBfFdxadNEJTQ8GW0fPtOIVwDZtJlNwi7ep58uR9Zld6Zo7FLRSzx -PtzBP6eEtwMJtVCk6PFluA7MY7k4c/TUW8bK0m9ybHIB8nqKuSWhZQBLdOhISyBz -/12xzX3An1NUpUaJnnD6ypEyfd8nZC0oAFC6+SAUMBWxcWYvhE5zcMaZQ3YtJUiC -0gR5d0Z1sjPYsq4KPow7IaTnzu3+0nLjZUHdU9RMfehJAxgBm3x0 ------END CERTIFICATE----- \ No newline at end of file +MIIDOzCCAiOgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipcwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTUwMzlaFw0zNDAxMzEy +MTUwMzlaMCUxDTALBgNVBAsMBENOQ0YxFDASBgNVBAMMC2V4YW1wbGUuY29tMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzLB7CGHcRrfMLHqMRkELOCvg +H7858LlvkG58fnhk0vjkmi6i/bvNSXV8YXngFbFY0bc+aj91/Vbu+YEkFPgFYRae +vPVTUeNb2jrCr19nc1QmlghML+ugnpAKKfPb8gen5teLkq/SLrIsR7ur347s3thW +ROcXMOZkmDMrKik2zOAzI1xSVxYOZsNWAvc5Id8wllGXZonWn8DgejIiZFWbAEIw +MHOaGg+CP7mwnZlq/AjmG28FJd9pPrdHkH4/XL3ePke7gWFTVrtQTYNEajWWZSDO +BFaJ3NSfnje6BqOe4y68H+IwjpHINsHel+tgrrkB4cHXUHpOChrYTCVCB+0PUwID +AQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cu +ZXhhbXBsZS5jb20wHQYDVR0OBBYEFEbKWjY1gwvxJusV+M5wUn+7MKR5MB8GA1Ud +IwQYMBaAFCS8xkbI19UdTehNQBJJ9iJPhqs1MA0GCSqGSIb3DQEBCwUAA4IBAQAo +873zVMG6tfoPRUZ/kEJcPEIaLmaolALyLEx3sZHe/B8hszuuMecBEfa02HwTSlzq +fKrkME95LGE9D+4hxyPEqPTqruESShUvBFQIoTQxePAhhUG9icF4gqUpYvRHXMiR +xIyfH3/KojDlBXRfDOaoXEXshiXfcYqbeh2qFdaoN24Vyh6lkNa2K3SUDAtKVFiF +jjBtNXuH/IJ3EWbs5AOOy98QtBMlT7kmummJVeaRR4QUfnzFrlj5nMSIopoxDm4N +QeoQSC+63fce6ZJLGQqEFQNR6howBcDQ/8fMR/oLsJ1Hr9VshIsu3kGTk4RUqHI9 +ipGt1UTvVf/uMUzA3yoC +-----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/server-key.pem b/test/configs/certs/tlsauth/server-key.pem index 38615ad58d9..ea0dd7873ce 100644 --- a/test/configs/certs/tlsauth/server-key.pem +++ b/test/configs/certs/tlsauth/server-key.pem @@ -1,27 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEAv+qCDnauYy08xfxu5S9Lm9oRlhXn0XUS1S73WaA4Sgv+QYYR -rDmsgjXTwiJ0VEDO+IUBXtqO1NOUxfpjifGX6rTdtab+Kxe9hztCacSnaDKUd887 -qmDqmCXwKGxEsz7loEliUMs/vlEsyTwpU6NjcIlKqlsOC8Jcyd4zffpPFmQuOnXz -r59lHiX5/kAQUoAPdJHjk6AEwa8lEgMjuOWiilkQxRIHPUPiVgs0l4AMqJcdW5BP -zbKSWpn4mSNPQIY/2hTWNdroe6qOAqWbtxtV8IkaQQGoEhavRRdgb92LGv9tK3h8 -qW2x+Q/Ac+zhRdXdUgDN2I2FhIh3aTepS29y+QIDAQABAoIBAQCxRXSM16ONiKOy -XdIxcNZuR6gm8mUHXRTgRlnEN/LGsv1QmP6KD1wBiqbnk9vQV2zWskTp0QhOHoI1 -vWtkZ/zjl92ThYURWQSAfYSDHltkLBRn9swuPQd1MtX7AMcUquyAekiOSK/ApEqy -NxgVYb7gnHCTmzgGNKpw7QazPxr+mkEOSKVNZvygNITPA9lvuCZ8Ky8ctIqOJOPB -wbIzHx3R18RPB++NMJ9T+mE+2vNhfJ4z6qVnOGNDle4Z3R0+JJog9G/57xlOd5HW -aTkM4HkA2sBsqfi3MA/DfhRAg+I8d5NFHEXc7pCX1NdaiDKVNPE0gd8mDd5BhJit -/lF09kQBAoGBANO06LLQepJnM5MNxdI2SYAxUZ/imlWN50ffuZcOmOgn7J9/3Gv2 -VUcJ01JeJwir8Vf+dl/d/F/fYQ6npss+XNpJ+7BB6OsNI6CMC87VH1YlT3QqG5qg -lAnMWK6YCxD5lu44P0dbR+R/NmW9yf1UPJGS3CKNoGWW9njb3isc5/iJAoGBAOgR -mvlCa4WD9WRRxeMW5RoWWZ+2a3OUm5Iy1uENsKCZ4gwWKoy96lMmfUh01sGgFewh -wgnX1hbkZe+YDqLQl2FpeBu1fzic1pCSVFBByaI2HZjufoGuo84ny9J++yoNABge -FuQSWXB7JT2v4SKK0mjV1LXzJawPHmJXzJayCCrxAoGBAJ13UCXAn4rJrDjS47MJ -of3xsQ7FU5oTJFX3eGl8+Aqlt4Cjb+X1oVRnYIFBerMegTK8GHwR9yewVNa7qHo/ -9nx+zvA49e/vI/LEd/vt1ZMTyVdUApgunC31injCqmiD3NlviNGgeYbhgCqI0fbV -cv+sRoSE5yro8IbQsx1KMNhRAoGBAMo23QUpXSuAKol5v6b7QjKTGxFSERsreMvR -xO9h0HCA5jmF7xmoOtCtjyldtewOJEwXtk6BZimYZ0J5CvfQLrhRALmUUwDvmP9s -ok80pA/We7/QwScbF90BTFdlElI39ccOIQAnBQxAIdk4skI5GNME0E6jSkY8/krP -GpSNGRThAoGAQsILz87TekVmyZuSGObkbnbFtvFKSfOSCuYuHIJjKyd4lUZi/CDY -iEtnXKk1UHT00zvliRGMrqA6++ePBZuMtN+8oRrGyfVEgQaHlNKlBMNvIrMgbZj0 -UGRJsOFDn2+kGsOvMdLMZKMzfqKk2mNefl10Ti0wbNXpG0LQC7JlhZQ= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDepPh4tWYw66pP +GCEgpYzBTW36/s7cRnsuqhkiUolc5RB//JYaRz/sbIlY7tM8Z21nmjU0IXvlKPUt ++6Winv7VqDU7S1gOssnIPbVhXGCb4fC3JGjQ2qD/pO5WfoUh8yYph9tFg07+zMyQ +XgJiE8JglKCeD7kjna0/lkk63ddQP0npbH5943oMvHDj7UjJyBdE8nWX1LXWAmYJ +Wxym6MDfjWTVnyzwqm8fkl/BHp9fh98Nf9U6/gpq0ckRUHL4prhMw0ChhBVjkBQ8 +CrXrp82TQZbeFPm3uWTV3xf5bnbeO87dgiXP47yvqVFSQI2cFyo2JzEq6LVAeVPT +dKzENXyFAgMBAAECggEAHlLdvKMIPhV65rbknCuwFgvTtOHLjtjSojJspe4T42EX +dDcUwpN9s1e9BS3R+2Ii1n98S5Nb6oQ/kHm7v4BkOPll9qN2ZNoY/XraH16Tkeed +/3OoCvob/3WZOJKW017ojbOBO+B8e9us6OTE8lK6oKjdj2mYz68ED6sKYkggsT8M +ah4KkzrX8xNiQwMRs0VMfXb1M8DUd+nWcamWdUytf3ZPFWxymzYzjjxDuisK4faV +Fuqwk2FRtOuyMeygC7KuPwjBtqXuFuHLYtOlibs5ZFG9Nwd9lWQEaVCnOlgW35zz +CVkdszfCgimCL/K42vcmbdnI3GUO/g3szw9Vy95sAQKBgQDx/PHZks4FdxXSkuxZ +u3Qjh5d/sG9NMK0QUNgF/MnORQjvmVfQOSXObiMxoXBBNW25St00mO0W+ZzXee5z +3ANw6cPgSXla29RKC4yVuAXwOn/kPnuASQxdoGWEKuZJqUKbpK+FDgFbK9cZlSd/ +VNS9eW2j00CCqXK0FQ7FvfCoBQKBgQDriUonm0EwCoNrBlbllOynFYjFwg2yu/ti +H/1SAdmTcD+NZFRwhBmSKxni2rYyMoyqkHN93t8Sz8ELIK4sEsA/XUpEioeGIwMG +HQSRga2XYUVafuI/TQRHeMke703rtbIN7dkjEiWsM1gHoPcTRxZU9hRGq0p89GHq +zOf4tuAqgQKBgA3ceVYHLLnvalaXh+ZT8IEggTMVPirjwOYQW29sXXrtRWfEFt2c +iGfcszNilfWGQ/S7LxSWNe58+dj16QzF64SKP2gXjVYBBZYAN1tUITLzhuPiGFzu +0kCCsY3yjyJlCaW0t0Ed3kIErtuOSabnixAXZopdzXIulp1uH1yAVsqpAoGBAL+7 +ua62FoGp/ULRHUm0SlTVFcqN5iK6Ha/KBKeOM/Ruan2Jz6bsEfjHt0HQ8oG4XoO2 +JR2woHyqvCV3y/C6rt6l7YAQGLRbqel/E6nzG0FggFljcn8/DZ20uFvDR/X5qWDn +XlvLOPmNrjo/kQGTW5172BOS+obvVQoTFT6Ed8SBAoGBAI5802KpwzgN9IjPM1Zf +rBZ6PshZ14PFxeQ2tOhvFcr+N+ceSY19lex+UyQn7F214Kz2U3n35zgNdLhdHLK6 +WokXp21WDqeg8n1cUrQ43yWqVX8cjdNx3P84B5Mdnb0nhbM2smUSuhrO+fOzr37A +JegHloScSH3BcemRFfK31+ba +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/server.pem b/test/configs/certs/tlsauth/server.pem index 7bdd9a73b70..2ed37dca31e 100644 --- a/test/configs/certs/tlsauth/server.pem +++ b/test/configs/certs/tlsauth/server.pem @@ -1,22 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDoTCCAomgAwIBAgIUKrcs29uAsjrZ53kR88U2UAiz2FgwDQYJKoZIhvcNAQEL -BQAwTDEkMCIGA1UEChMbU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuMRAwDgYD -VQQLEwdOQVRTLmlvMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTkwMjE4MTgxOTAw -WhcNMjQwMjE3MTgxOTAwWjAwMRowGAYDVQQLExFOQVRTLmlvIE9wZXJhdG9yczES -MBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC -AQEAv+qCDnauYy08xfxu5S9Lm9oRlhXn0XUS1S73WaA4Sgv+QYYRrDmsgjXTwiJ0 -VEDO+IUBXtqO1NOUxfpjifGX6rTdtab+Kxe9hztCacSnaDKUd887qmDqmCXwKGxE -sz7loEliUMs/vlEsyTwpU6NjcIlKqlsOC8Jcyd4zffpPFmQuOnXzr59lHiX5/kAQ -UoAPdJHjk6AEwa8lEgMjuOWiilkQxRIHPUPiVgs0l4AMqJcdW5BPzbKSWpn4mSNP -QIY/2hTWNdroe6qOAqWbtxtV8IkaQQGoEhavRRdgb92LGv9tK3h8qW2x+Q/Ac+zh -RdXdUgDN2I2FhIh3aTepS29y+QIDAQABo4GWMIGTMA4GA1UdDwEB/wQEAwIFoDAd -BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNV -HQ4EFgQUsi295rOWX4epJovs+SFWKL0M9NEwHwYDVR0jBBgwFoAUbwbb4b9Hyi/J -dmgKO0hyj272GsswFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUA -A4IBAQBt+JD/POQavswoTezh7YtXaw2F3DFhOz3yGvcKXBG/wRafWPuEZCMjY1cx -EgjOxA5kHPcsMNjRtQGAuV+1EymWUhJCyUq109D4eSNzR3IwXP3FhVtZ71XxCl+c -qUatFWvfGaU30TAHwB/QcXhLsliEMOaKRr3yTgM2BaTjsaBSA0/s/JwQiHvF0n1R -YYP4G7BON8IzhjsD38KjUrlAXT24VzCcDyFDk2c434jQwOZVxqHidB3lBlOmjX1X -eRJcJx/YkE7ej70R2f2WhQfZkgeF3iX8Xa5a10vjIqXcI9DkVb3GwAjx6+XDekJI -j4uv6u+kCJxABz3EyggU4BEPEPNL +MIIDKjCCAhKgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipUwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MzJaFw0zNDAxMzEy +MTQ4MzJaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAN6k+Hi1ZjDrqk8YISCljMFNbfr+ztxGey6qGSJSiVzlEH/8 +lhpHP+xsiVju0zxnbWeaNTQhe+Uo9S37paKe/tWoNTtLWA6yycg9tWFcYJvh8Lck +aNDaoP+k7lZ+hSHzJimH20WDTv7MzJBeAmITwmCUoJ4PuSOdrT+WSTrd11A/Sels +fn3jegy8cOPtSMnIF0TydZfUtdYCZglbHKbowN+NZNWfLPCqbx+SX8Een1+H3w1/ +1Tr+CmrRyRFQcvimuEzDQKGEFWOQFDwKteunzZNBlt4U+be5ZNXfF/ludt47zt2C +Jc/jvK+pUVJAjZwXKjYnMSrotUB5U9N0rMQ1fIUCAwEAAaN2MHQwMgYDVR0RBCsw +KYIJbG9jYWxob3N0ggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tMB0GA1Ud +DgQWBBS/WuyC1fvhLL8rnKIXWFdQFnGpozAfBgNVHSMEGDAWgBQkvMZGyNfVHU3o +TUASSfYiT4arNTANBgkqhkiG9w0BAQsFAAOCAQEAVgdcRJgo95MEDgEACekziOHn +n86DdgNin4FDkL7Y2sBDrpej4B0jKPm2H/M4qdB2Z17Ru2TdcXOyk/sMc395GHsN +BAdKuAcysvQ+USR3UXasJmC/CvoKGBOmFf9/Jor8U4Rs01bkXSd6pW8ytT3kyMak +3r5tNugzRxpJvVDgjHlUkfhBoLeeCr+k1cN1OvR4cFhY6vxqS6GBdopFGC3DlnTL +LPetNhQCd+r2mH1RT/56aLLRawy76GkBEZm/+mg+mYjxN3J1hWibouF4ccutvxtt +h2/4PJNsXv5yt4wibazFixJ843KPdfw6pafXbYZsvvgNfvLrpp8beCUwHEYq+w== -----END CERTIFICATE----- From 0447a6fd302d937837f1791eeb1873d8e90e1f8f Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Sat, 3 Feb 2024 14:06:06 -0800 Subject: [PATCH 04/21] test: update certs for TestTLSRoutesCertificateCNBasedAuth Signed-off-by: Waldemar Quevedo --- test/configs/certs/tlsauth/server-key.pem | 52 +++++++++--------- .../certs/tlsauth/server-no-ou-key.pem | 55 ++++++++++--------- test/configs/certs/tlsauth/server-no-ou.pem | 36 ++++++------ test/configs/certs/tlsauth/server.pem | 35 ++++++------ 4 files changed, 89 insertions(+), 89 deletions(-) diff --git a/test/configs/certs/tlsauth/server-key.pem b/test/configs/certs/tlsauth/server-key.pem index ea0dd7873ce..a9077a68aee 100644 --- a/test/configs/certs/tlsauth/server-key.pem +++ b/test/configs/certs/tlsauth/server-key.pem @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDepPh4tWYw66pP -GCEgpYzBTW36/s7cRnsuqhkiUolc5RB//JYaRz/sbIlY7tM8Z21nmjU0IXvlKPUt -+6Winv7VqDU7S1gOssnIPbVhXGCb4fC3JGjQ2qD/pO5WfoUh8yYph9tFg07+zMyQ -XgJiE8JglKCeD7kjna0/lkk63ddQP0npbH5943oMvHDj7UjJyBdE8nWX1LXWAmYJ -Wxym6MDfjWTVnyzwqm8fkl/BHp9fh98Nf9U6/gpq0ckRUHL4prhMw0ChhBVjkBQ8 -CrXrp82TQZbeFPm3uWTV3xf5bnbeO87dgiXP47yvqVFSQI2cFyo2JzEq6LVAeVPT -dKzENXyFAgMBAAECggEAHlLdvKMIPhV65rbknCuwFgvTtOHLjtjSojJspe4T42EX -dDcUwpN9s1e9BS3R+2Ii1n98S5Nb6oQ/kHm7v4BkOPll9qN2ZNoY/XraH16Tkeed -/3OoCvob/3WZOJKW017ojbOBO+B8e9us6OTE8lK6oKjdj2mYz68ED6sKYkggsT8M -ah4KkzrX8xNiQwMRs0VMfXb1M8DUd+nWcamWdUytf3ZPFWxymzYzjjxDuisK4faV -Fuqwk2FRtOuyMeygC7KuPwjBtqXuFuHLYtOlibs5ZFG9Nwd9lWQEaVCnOlgW35zz -CVkdszfCgimCL/K42vcmbdnI3GUO/g3szw9Vy95sAQKBgQDx/PHZks4FdxXSkuxZ -u3Qjh5d/sG9NMK0QUNgF/MnORQjvmVfQOSXObiMxoXBBNW25St00mO0W+ZzXee5z -3ANw6cPgSXla29RKC4yVuAXwOn/kPnuASQxdoGWEKuZJqUKbpK+FDgFbK9cZlSd/ -VNS9eW2j00CCqXK0FQ7FvfCoBQKBgQDriUonm0EwCoNrBlbllOynFYjFwg2yu/ti -H/1SAdmTcD+NZFRwhBmSKxni2rYyMoyqkHN93t8Sz8ELIK4sEsA/XUpEioeGIwMG -HQSRga2XYUVafuI/TQRHeMke703rtbIN7dkjEiWsM1gHoPcTRxZU9hRGq0p89GHq -zOf4tuAqgQKBgA3ceVYHLLnvalaXh+ZT8IEggTMVPirjwOYQW29sXXrtRWfEFt2c -iGfcszNilfWGQ/S7LxSWNe58+dj16QzF64SKP2gXjVYBBZYAN1tUITLzhuPiGFzu -0kCCsY3yjyJlCaW0t0Ed3kIErtuOSabnixAXZopdzXIulp1uH1yAVsqpAoGBAL+7 -ua62FoGp/ULRHUm0SlTVFcqN5iK6Ha/KBKeOM/Ruan2Jz6bsEfjHt0HQ8oG4XoO2 -JR2woHyqvCV3y/C6rt6l7YAQGLRbqel/E6nzG0FggFljcn8/DZ20uFvDR/X5qWDn -XlvLOPmNrjo/kQGTW5172BOS+obvVQoTFT6Ed8SBAoGBAI5802KpwzgN9IjPM1Zf -rBZ6PshZ14PFxeQ2tOhvFcr+N+ceSY19lex+UyQn7F214Kz2U3n35zgNdLhdHLK6 -WokXp21WDqeg8n1cUrQ43yWqVX8cjdNx3P84B5Mdnb0nhbM2smUSuhrO+fOzr37A -JegHloScSH3BcemRFfK31+ba +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAgrLiRyYcP54+ +AlElXoMjhOMg+VZjkVnC4CVXk+3jmKqAzI6bum9QHBnSdN/ULaPFU4Vgoqt1Puv8 +J9snNlSSE4CcgCCmWihFIwSpIaXW/GWCZVvCQDn3QxAZt48vTtEI27tMsRTCk/tV +kuShJ7OxrJyXcntBWbS+TeSOMVIF/v6lqWGjaDauPK0cpesa/qLElaHNlIV0+pM+ +5b3LIcEpuaECVwu5n5c9m/qdX7ZyPF2x8Z2I0zO5nLyFxb3WolPjok/qnZbhC/Gp +OX459F6yCVBmjkakaJocV1Ue7V0dFB7u3aYW0dwa+7Zb0613ZHMWKfAOzJKOiZC5 +xYXBxO4DAgMBAAECggEAA+ILzXOfVneHPHuO4acuRvoIG/yoT1OgHcWoPl16ywBX +BBUPYh9Yqm/lEQrDDwRCpuX54HEEVO+WRY/gpEHv/I6z8k3FTgWu66h6qhZGW/eo +U02yRzdFBz9AV/xzWpvUEvUq3Uk/GkBvRxKyiN7RPMODpXktXxKTo9Kg/XTYnjIb +2RWzKdoeptX6q/RRdPkBXxYV5XIaZLXPy+VaGgI7QcwApmCcin20c5smtLmGE7TV +4hh6wxPwGEiDkfMQGnqWd+VdzbnZjZF2/4ZVCNAbJJaFffTxnoC0uGjE3ImAztmT +TgmFbdfO/5qHTkJmfZaAbmo461k1O2Vd/p6LCN1mhQKBgQDokSJxOMuYurf1STAb +om6E5lGXAYR/2TsBxzfWPvUlZDzaSZ7IU+/zpv9GTATc0mvdIpnT2kZvRoz7nbbR +LFGjc2d1ITRBuDEX+qDkX1puFyhoYPyaz0AuC3AMqg28e3Z1I8nq3Op1TUKAMr9d +ZdOfRUcL65OM+YHPZ197AVwC9wKBgQDT6FgGceqBr3XJDikjU+kJfyrZrcE7sfrE +3RIV6uMj6Myw4WpUpuH7hwqQ0xAir8UKHG6SDScymXtrJCoG1xJmGGDyoLt80VsF +fXVhzL8rhIl48TVWptRUiIeZ2dXvianLkLPlLhVUcTThpjjAAbnNYhod1bHygX12 +w+Bd8lkeVQKBgBYLS7x3qbS8Xht96HV2HAu02R77IdgMey9b9sr0BMCak7oNKGPM +sP3jYmcDZaKYv2iikvolwm9hvJNNC7sf/E0F71SG5TEliGHBe+apsySkRUw/hTIX +WvoCU4ifxdWLzlqkHcuJTR/5RshoBwOPV1PNeUKD/eRq8gb6wW4jXtlZAoGBAK2o +C3MEqcQrUSA53ZaY7jGdKDWJQgC0oyfvbyHNAuVro0sU/3lt5WWmTg9PGDsExjm6 +ARbpdoTt6Ilt8o72c5p9Qf2zoNHyE2CVZruF+egkzi/xo99mCj1YQZ/gN4T80MwE +wpf+wvYXa9m7yWf4QhbA3VwzwodUfMf2T4lN0KCdAoGBAM04ecwBXCNIXzTgJDeK +zxfI95lphcNywdhUNw6KJPDg4ozPu72XrVToAtl7l6+2RPygNhSj117002AAgz30 +l0cfZoogXmIhZt+R39/2e9y31k+GDdEmNtPLsvbLPQ+Ttnab87pUABn4UC1XXGhF +Vt/jfLEw8ZDjB+aO4kFyZPIQ -----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/server-no-ou-key.pem b/test/configs/certs/tlsauth/server-no-ou-key.pem index 39ce9f2c21f..ea0dd7873ce 100644 --- a/test/configs/certs/tlsauth/server-no-ou-key.pem +++ b/test/configs/certs/tlsauth/server-no-ou-key.pem @@ -1,27 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA330XGNotGWellHcjsr2O8rL51Htvk6wtJAep8wsPL7KtZYqe -0GL6No3WSw/JVTaIf27e45tBTLSPrVk0qGzkBAXxpy6ywdWmVL2NhoZ5E0jeABTb -Ud3Lj6zL+ggM+BQq2xJh85fytSNJ9nEcPBHzbwGB6baKpE84872qQ2jH8vXzdEHI -kWseJGItJ5JAcXGBQ0DqpUTUhI9Nww+EWjOYbiFZg2cy8ZMgOwReEgioQrwrN5i1 -PPeIpRKk23cQjeDa50J6uQjj6rcAO8PcQLjB4VCuSo4y4RSCLK4N5JZm01oOndLZ -Viyo14Rk87zNfQVKAlBm+wGfoYfbPNCu2oDUDwIDAQABAoIBAQCKW8n509CJ4tXq -pRuPo7Uk4dKzTjvUY4qKvMflNJqRQAADVh9eCXi4X3UkgUB2pc05f20z4cE6eKpe -elSUVN2Q9BEYHFwEjn1sBvHgL84zNzBhQohJFZPZffPF9kf5KZsihy3m/dH/fDpk -/L+rHL3lojxIcX9Bm945X9lR5EOtUJhkZ2xJI37VHNN1iWvNcNwls/Ud+lwsxndt -Pj8tjPZ1e0ypMSJllgKi8X4+JBIfcOL7v5OmNwOx5YMuUYDRWT35I3Tz/M12o0BB -nrrDBeTRXq/netqBWEa15id4g7xSGvZQKoNOpqDjtQZU851DRQniM1hqjTZR9qj+ -Pdj3/41RAoGBAOH+OBNogdvlADW4lUCcbF0cKktNwsv4hsRFyRUs3Yz91ESLhCc8 -KKRdqTO1SB3wdt8Q+Dxfsdorjf51CX1hjTk7lHL8v0GhU3zjOsanACd0pDMQK1lv -fQVKvZHZms5UZD0G27l6yS/Dxg7fkDUkTcR1n0t4cJqpjP0OKU+mkyMZAoGBAP0p -vkP7D4ICO2el023xK14SR1I4RbgEO9h3+IbdWsOjhioB/WjvWsWBJYa5ihFYCaGN -x3nEl1J8xCy6ADjbtvSYcgTWKjcsABY6adlwmoBE/UlwFrr0VA5+qJCyRK/zJMjr -Y5pICmC2rV6T+C+jpEOGNd06fN+lbNHfcAXKTv1nAoGAe2Cxto7Qjn9IDQwXl62O -T4rn4DK0zWyCDrdWn1PeJHITJ9TPMihau9lSXaNzmrzD+OYnz7Yiv8wVejzlEGlo -kz1evyQTOj5b+QuI9BkKMYAxgJssP2hpZbE3K2AUbt6N1u9el7VcDtKf11DgRtLq -Df51F9vKBfXYvfK0RQLYw0kCgYBioanEGINBNpdoWT3XXpdzzhFFYjEfcV7ThmIo -QQNEp2f049OT13T478jsBUtaWH9gFrm5ojMGax+PAWRmwos0HlSFt964ogbioh1t -HqbDBJ3dx7LDYb+B6izIOvvxxPv232ZtzFVmuqUu7N1LyiiMOjSwHUJba7rKxY+C -YgCGTwKBgFLUxkEDszCMPKQRb2G7rt/D5Hat7gsBvkYifa8wPA57v20johtKDlGF -lo1AolMCsebwS9IwwMO5taenJrWVNnw+YeHkr9E7sHoOlxRWjJImxs30cGJAEdQg -xMANEnpb8OQd4t47dpBgMh5KXqgQqg3kGPO55ZJlb1wfQ1HqWLii ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDepPh4tWYw66pP +GCEgpYzBTW36/s7cRnsuqhkiUolc5RB//JYaRz/sbIlY7tM8Z21nmjU0IXvlKPUt ++6Winv7VqDU7S1gOssnIPbVhXGCb4fC3JGjQ2qD/pO5WfoUh8yYph9tFg07+zMyQ +XgJiE8JglKCeD7kjna0/lkk63ddQP0npbH5943oMvHDj7UjJyBdE8nWX1LXWAmYJ +Wxym6MDfjWTVnyzwqm8fkl/BHp9fh98Nf9U6/gpq0ckRUHL4prhMw0ChhBVjkBQ8 +CrXrp82TQZbeFPm3uWTV3xf5bnbeO87dgiXP47yvqVFSQI2cFyo2JzEq6LVAeVPT +dKzENXyFAgMBAAECggEAHlLdvKMIPhV65rbknCuwFgvTtOHLjtjSojJspe4T42EX +dDcUwpN9s1e9BS3R+2Ii1n98S5Nb6oQ/kHm7v4BkOPll9qN2ZNoY/XraH16Tkeed +/3OoCvob/3WZOJKW017ojbOBO+B8e9us6OTE8lK6oKjdj2mYz68ED6sKYkggsT8M +ah4KkzrX8xNiQwMRs0VMfXb1M8DUd+nWcamWdUytf3ZPFWxymzYzjjxDuisK4faV +Fuqwk2FRtOuyMeygC7KuPwjBtqXuFuHLYtOlibs5ZFG9Nwd9lWQEaVCnOlgW35zz +CVkdszfCgimCL/K42vcmbdnI3GUO/g3szw9Vy95sAQKBgQDx/PHZks4FdxXSkuxZ +u3Qjh5d/sG9NMK0QUNgF/MnORQjvmVfQOSXObiMxoXBBNW25St00mO0W+ZzXee5z +3ANw6cPgSXla29RKC4yVuAXwOn/kPnuASQxdoGWEKuZJqUKbpK+FDgFbK9cZlSd/ +VNS9eW2j00CCqXK0FQ7FvfCoBQKBgQDriUonm0EwCoNrBlbllOynFYjFwg2yu/ti +H/1SAdmTcD+NZFRwhBmSKxni2rYyMoyqkHN93t8Sz8ELIK4sEsA/XUpEioeGIwMG +HQSRga2XYUVafuI/TQRHeMke703rtbIN7dkjEiWsM1gHoPcTRxZU9hRGq0p89GHq +zOf4tuAqgQKBgA3ceVYHLLnvalaXh+ZT8IEggTMVPirjwOYQW29sXXrtRWfEFt2c +iGfcszNilfWGQ/S7LxSWNe58+dj16QzF64SKP2gXjVYBBZYAN1tUITLzhuPiGFzu +0kCCsY3yjyJlCaW0t0Ed3kIErtuOSabnixAXZopdzXIulp1uH1yAVsqpAoGBAL+7 +ua62FoGp/ULRHUm0SlTVFcqN5iK6Ha/KBKeOM/Ruan2Jz6bsEfjHt0HQ8oG4XoO2 +JR2woHyqvCV3y/C6rt6l7YAQGLRbqel/E6nzG0FggFljcn8/DZ20uFvDR/X5qWDn +XlvLOPmNrjo/kQGTW5172BOS+obvVQoTFT6Ed8SBAoGBAI5802KpwzgN9IjPM1Zf +rBZ6PshZ14PFxeQ2tOhvFcr+N+ceSY19lex+UyQn7F214Kz2U3n35zgNdLhdHLK6 +WokXp21WDqeg8n1cUrQ43yWqVX8cjdNx3P84B5Mdnb0nhbM2smUSuhrO+fOzr37A +JegHloScSH3BcemRFfK31+ba +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/server-no-ou.pem b/test/configs/certs/tlsauth/server-no-ou.pem index 5aebb7d2f27..2ed37dca31e 100644 --- a/test/configs/certs/tlsauth/server-no-ou.pem +++ b/test/configs/certs/tlsauth/server-no-ou.pem @@ -1,21 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDhTCCAm2gAwIBAgIUbXHf4iAemXfIpLSWpRMkEVsdjy8wDQYJKoZIhvcNAQEL -BQAwTDEkMCIGA1UEChMbU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuMRAwDgYD -VQQLEwdOQVRTLmlvMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTkwMjE4MjE0MjAw -WhcNMjQwMjE3MjE0MjAwWjAUMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDffRcY2i0ZZ6WUdyOyvY7ysvnUe2+TrC0k -B6nzCw8vsq1lip7QYvo2jdZLD8lVNoh/bt7jm0FMtI+tWTSobOQEBfGnLrLB1aZU -vY2GhnkTSN4AFNtR3cuPrMv6CAz4FCrbEmHzl/K1I0n2cRw8EfNvAYHptoqkTzjz -vapDaMfy9fN0QciRax4kYi0nkkBxcYFDQOqlRNSEj03DD4RaM5huIVmDZzLxkyA7 -BF4SCKhCvCs3mLU894ilEqTbdxCN4NrnQnq5COPqtwA7w9xAuMHhUK5KjjLhFIIs -rg3klmbTWg6d0tlWLKjXhGTzvM19BUoCUGb7AZ+hh9s80K7agNQPAgMBAAGjgZYw -gZMwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD -AjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBSubJojZMv2Ithfumfx12lAhWyaSjAf -BgNVHSMEGDAWgBRvBtvhv0fKL8l2aAo7SHKPbvYayzAUBgNVHREEDTALgglsb2Nh -bGhvc3QwDQYJKoZIhvcNAQELBQADggEBACGDpZ9oxuuCB8ujf7YXMPw7Ae1WH7DL -pBtPNL99bnPLQd/6iv12TWBFal5xSTCxhN/exAqDEk36zCKIEk2LvY/VJHP8Y2si -A79PqrlzSptOxeEuQRZnk+FiWYLwelkvL66TnpUW3QwdCCj/vodabDlaq6eSMEF+ -+kwxWZUixYkWASwuPSd7xjgNNBvIjeGvZIZvyjgTwyzPx/hSEMET68lRAoeE1S7S -IHqLPp/UwvvI9qMzBzkOz/XOmptB2fG3a5BLefEErcjVqfqYNAAA5V58+pyRrCSO -nX8UaSmy42aHryV0eelTLIxFSeSOHycpEhY2Hxq75JG8eSNfRd9rSt8= +MIIDKjCCAhKgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipUwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MzJaFw0zNDAxMzEy +MTQ4MzJaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAN6k+Hi1ZjDrqk8YISCljMFNbfr+ztxGey6qGSJSiVzlEH/8 +lhpHP+xsiVju0zxnbWeaNTQhe+Uo9S37paKe/tWoNTtLWA6yycg9tWFcYJvh8Lck +aNDaoP+k7lZ+hSHzJimH20WDTv7MzJBeAmITwmCUoJ4PuSOdrT+WSTrd11A/Sels +fn3jegy8cOPtSMnIF0TydZfUtdYCZglbHKbowN+NZNWfLPCqbx+SX8Een1+H3w1/ +1Tr+CmrRyRFQcvimuEzDQKGEFWOQFDwKteunzZNBlt4U+be5ZNXfF/ludt47zt2C +Jc/jvK+pUVJAjZwXKjYnMSrotUB5U9N0rMQ1fIUCAwEAAaN2MHQwMgYDVR0RBCsw +KYIJbG9jYWxob3N0ggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tMB0GA1Ud +DgQWBBS/WuyC1fvhLL8rnKIXWFdQFnGpozAfBgNVHSMEGDAWgBQkvMZGyNfVHU3o +TUASSfYiT4arNTANBgkqhkiG9w0BAQsFAAOCAQEAVgdcRJgo95MEDgEACekziOHn +n86DdgNin4FDkL7Y2sBDrpej4B0jKPm2H/M4qdB2Z17Ru2TdcXOyk/sMc395GHsN +BAdKuAcysvQ+USR3UXasJmC/CvoKGBOmFf9/Jor8U4Rs01bkXSd6pW8ytT3kyMak +3r5tNugzRxpJvVDgjHlUkfhBoLeeCr+k1cN1OvR4cFhY6vxqS6GBdopFGC3DlnTL +LPetNhQCd+r2mH1RT/56aLLRawy76GkBEZm/+mg+mYjxN3J1hWibouF4ccutvxtt +h2/4PJNsXv5yt4wibazFixJ843KPdfw6pafXbYZsvvgNfvLrpp8beCUwHEYq+w== -----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/server.pem b/test/configs/certs/tlsauth/server.pem index 2ed37dca31e..0e9f92ae971 100644 --- a/test/configs/certs/tlsauth/server.pem +++ b/test/configs/certs/tlsauth/server.pem @@ -1,19 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDKjCCAhKgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipUwDQYJKoZIhvcNAQEL -BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MzJaFw0zNDAxMzEy -MTQ4MzJaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQAD -ggEPADCCAQoCggEBAN6k+Hi1ZjDrqk8YISCljMFNbfr+ztxGey6qGSJSiVzlEH/8 -lhpHP+xsiVju0zxnbWeaNTQhe+Uo9S37paKe/tWoNTtLWA6yycg9tWFcYJvh8Lck -aNDaoP+k7lZ+hSHzJimH20WDTv7MzJBeAmITwmCUoJ4PuSOdrT+WSTrd11A/Sels -fn3jegy8cOPtSMnIF0TydZfUtdYCZglbHKbowN+NZNWfLPCqbx+SX8Een1+H3w1/ -1Tr+CmrRyRFQcvimuEzDQKGEFWOQFDwKteunzZNBlt4U+be5ZNXfF/ludt47zt2C -Jc/jvK+pUVJAjZwXKjYnMSrotUB5U9N0rMQ1fIUCAwEAAaN2MHQwMgYDVR0RBCsw -KYIJbG9jYWxob3N0ggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tMB0GA1Ud -DgQWBBS/WuyC1fvhLL8rnKIXWFdQFnGpozAfBgNVHSMEGDAWgBQkvMZGyNfVHU3o -TUASSfYiT4arNTANBgkqhkiG9w0BAQsFAAOCAQEAVgdcRJgo95MEDgEACekziOHn -n86DdgNin4FDkL7Y2sBDrpej4B0jKPm2H/M4qdB2Z17Ru2TdcXOyk/sMc395GHsN -BAdKuAcysvQ+USR3UXasJmC/CvoKGBOmFf9/Jor8U4Rs01bkXSd6pW8ytT3kyMak -3r5tNugzRxpJvVDgjHlUkfhBoLeeCr+k1cN1OvR4cFhY6vxqS6GBdopFGC3DlnTL -LPetNhQCd+r2mH1RT/56aLLRawy76GkBEZm/+mg+mYjxN3J1hWibouF4ccutvxtt -h2/4PJNsXv5yt4wibazFixJ843KPdfw6pafXbYZsvvgNfvLrpp8beCUwHEYq+w== +MIIDRjCCAi6gAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipgwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMjA0NTdaFw0zNDAxMzEy +MjA0NTdaMDAxGjAYBgNVBAsMEU5BVFMuaW8gT3BlcmF0b3JzMRIwEAYDVQQDDAls +b2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAgrLiRyYc +P54+AlElXoMjhOMg+VZjkVnC4CVXk+3jmKqAzI6bum9QHBnSdN/ULaPFU4Vgoqt1 +Puv8J9snNlSSE4CcgCCmWihFIwSpIaXW/GWCZVvCQDn3QxAZt48vTtEI27tMsRTC +k/tVkuShJ7OxrJyXcntBWbS+TeSOMVIF/v6lqWGjaDauPK0cpesa/qLElaHNlIV0 ++pM+5b3LIcEpuaECVwu5n5c9m/qdX7ZyPF2x8Z2I0zO5nLyFxb3WolPjok/qnZbh +C/GpOX459F6yCVBmjkakaJocV1Ue7V0dFB7u3aYW0dwa+7Zb0613ZHMWKfAOzJKO +iZC5xYXBxO4DAgMBAAGjdjB0MDIGA1UdEQQrMCmCCWxvY2FsaG9zdIILZXhhbXBs +ZS5jb22CD3d3dy5leGFtcGxlLmNvbTAdBgNVHQ4EFgQUcpTmScu/KZHhs0KbdxtN +GXndXBEwHwYDVR0jBBgwFoAUJLzGRsjX1R1N6E1AEkn2Ik+GqzUwDQYJKoZIhvcN +AQELBQADggEBAKdqG82z3JBim/hiGf4LZT90ZHhw7ngPT9HUV4jYRIk0ngJ37ogK +KCYW0UBCkugdf0elxcggjAsJZGlz+hW2j8MynEqJ9UU7jPPp4AKJqZHy5x49Y1iL +kFlJE5a3LFJUaVG4JeYMqTL2zDtoj+hk7QPPoz88moDUbOHg3HccObHlISelVPON +K/kvnJ2NfXImYkh7MusRxVuB4LcRRi5rwT0pOdtSPBCeSH96BOeCHTriPHGecgc4 +71tgSaELXPM1YnaM2WmXoGU1MZ7Dx6c2q97FI+SWgKfm7B1GQGyAghgKxlRyhfNj +UvCrbaZDInrMWpMo3+upIBWpHzfmJVvUcYI= -----END CERTIFICATE----- From 2fb30eda9ce83c8e93129f377c147450a15b31a4 Mon Sep 17 00:00:00 2001 From: Lev Brouk Date: Sun, 4 Feb 2024 03:55:05 -0800 Subject: [PATCH 05/21] [ADDED] MQTT: test/bench using external client --- .github/workflows/MQTT_test.yaml | 58 ++++- server/mqtt.go | 32 ++- server/mqtt_ex_test.go | 424 +++++++++++++++++++++++++++++++ 3 files changed, 491 insertions(+), 23 deletions(-) create mode 100644 server/mqtt_ex_test.go diff --git a/.github/workflows/MQTT_test.yaml b/.github/workflows/MQTT_test.yaml index fe73d6e8414..bfbd9210dab 100644 --- a/.github/workflows/MQTT_test.yaml +++ b/.github/workflows/MQTT_test.yaml @@ -1,15 +1,15 @@ -name: MQTT Compliance +name: MQTTEx on: [push, pull_request] +permissions: + pull-requests: write # to comment on PRs + contents: write # to comment on commits (to upload artifacts?) + jobs: test: - strategy: - matrix: - go: ["1.21"] env: GOPATH: /home/runner/work/nats-server GO111MODULE: "on" - runs-on: ubuntu-latest steps: - name: Checkout code @@ -20,18 +20,48 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: ${{matrix.go}} + go-version-file: src/github.com/nats-io/nats-server/go.mod + cache-dependency-path: src/github.com/nats-io/nats-server/go.sum - - name: Install deps - shell: bash --noprofile --norc -x -eo pipefail {0} + - name: Set up testing tools and environment + shell: bash --noprofile --norc -eo pipefail {0} + id: setup run: | wget https://github.com/hivemq/mqtt-cli/releases/download/v4.20.0/mqtt-cli-4.20.0.deb sudo apt install ./mqtt-cli-4.20.0.deb + go install github.com/ConnectEverything/mqtt-test@latest + + # - name: Download benchmark result for ${{ github.base_ref || github.ref_name }} + # uses: dawidd6/action-download-artifact@v2 + # continue-on-error: true + # with: + # path: src/github.com/nats-io/nats-server/bench + # name: bench-output-${{ runner.os }} + # branch: ${{ github.base_ref || github.ref }} - - name: Run tests - shell: bash --noprofile --norc -x -eo pipefail {0} + - name: Run tests and benchmarks + shell: bash --noprofile --norc -eo pipefail {0} run: | - set -e - cd src/github.com/nats-io/nats-server/server - go test -v -vet=off --run=TestMQTTCLICompliance - set +e + cd src/github.com/nats-io/nats-server + go test -v --run='MQTTEx' ./server + # go test --run='-' --count=10 --bench 'MQTT_' ./server | tee output.txt + # go test --run='-' --count=10 --bench 'MQTTEx' --timeout=20m --benchtime=100x ./server | tee -a output.txt + go test --run='-' --count=3 --bench 'MQTTEx' --benchtime=100x ./server + + # - name: Compare benchmarks + # uses: benchmark-action/github-action-benchmark@v1 + # with: + # tool: go + # output-file-path: src/github.com/nats-io/nats-server/output.txt + # github-token: ${{ secrets.GITHUB_TOKEN }} + # alert-threshold: 140% + # comment-on-alert: true + # # fail-on-alert: true + # external-data-json-path: src/github.com/nats-io/nats-server/bench/benchmark-data.json + + # - name: Store benchmark result for ${{ github.ref_name }} + # if: always() + # uses: actions/upload-artifact@v3 + # with: + # path: src/github.com/nats-io/nats-server/bench + # name: bench-output-${{ runner.os }} diff --git a/server/mqtt.go b/server/mqtt.go index 793d57d2736..f64c160ee58 100644 --- a/server/mqtt.go +++ b/server/mqtt.go @@ -234,6 +234,8 @@ type mqttSessionManager struct { sessions map[string]*mqttAccountSessionManager // key is account name } +var testDisableRMSCache = false + type mqttAccountSessionManager struct { mu sync.RWMutex sessions map[string]*mqttSession // key is MQTT client ID @@ -243,7 +245,7 @@ type mqttAccountSessionManager struct { flapTimer *time.Timer // Timer to perform some cleanup of the flappers map sl *Sublist // sublist allowing to find retained messages for given subscription retmsgs map[string]*mqttRetainedMsgRef // retained messages - rmsCache sync.Map // map[subject]*mqttRetainedMsg + rmsCache *sync.Map // map[subject]mqttRetainedMsg jsa mqttJSA rrmLastSeq uint64 // Restore retained messages expected last sequence rrmDoneCh chan struct{} // To notify the caller that all retained messages have been loaded @@ -1142,6 +1144,9 @@ func (s *Server) mqttCreateAccountSessionManager(acc *Account, quitCh chan struc quitCh: quitCh, }, } + if !testDisableRMSCache { + as.rmsCache = &sync.Map{} + } // TODO record domain name in as here // The domain to communicate with may be required for JS calls. @@ -1236,10 +1241,12 @@ func (s *Server) mqttCreateAccountSessionManager(acc *Account, quitCh chan struc }) // Start the go routine that will clean up cached retained messages that expired. - s.startGoRoutine(func() { - defer s.grWG.Done() - as.cleanupRetainedMessageCache(s, closeCh) - }) + if as.rmsCache != nil { + s.startGoRoutine(func() { + defer s.grWG.Done() + as.cleanupRetainedMessageCache(s, closeCh) + }) + } lookupStream := func(stream, txt string) (*StreamInfo, error) { si, err := jsa.lookupStream(stream) @@ -2243,9 +2250,7 @@ func (as *mqttAccountSessionManager) handleRetainedMsg(key string, rf *mqttRetai // Update the in-memory retained message cache but only for messages // that are already in the cache, i.e. have been (recently) used. - if rm != nil { - as.setCachedRetainedMsg(key, rm, true, copyBytesToCache) - } + as.setCachedRetainedMsg(key, rm, true, copyBytesToCache) return } } @@ -2269,7 +2274,9 @@ func (as *mqttAccountSessionManager) handleRetainedMsgDel(subject string, seq ui as.sl = NewSublistWithCache() } if erm, ok := as.retmsgs[subject]; ok { - as.rmsCache.Delete(subject) + if as.rmsCache != nil { + as.rmsCache.Delete(subject) + } if erm.sub != nil { as.sl.Remove(erm.sub) erm.sub = nil @@ -2770,6 +2777,7 @@ func (as *mqttAccountSessionManager) loadRetainedMessages(subjects map[string]st w.Warnf("failed to decode retained message for subject %q: %v", ss[i], err) continue } + // Add the loaded retained message to the cache, and to the results map. key := ss[i][len(mqttRetainedMsgsStreamSubject):] as.setCachedRetainedMsg(key, &rm, false, false) @@ -2960,6 +2968,9 @@ func (as *mqttAccountSessionManager) transferRetainedToPerKeySubjectStream(log * } func (as *mqttAccountSessionManager) getCachedRetainedMsg(subject string) *mqttRetainedMsg { + if as.rmsCache == nil { + return nil + } v, ok := as.rmsCache.Load(subject) if !ok { return nil @@ -2973,6 +2984,9 @@ func (as *mqttAccountSessionManager) getCachedRetainedMsg(subject string) *mqttR } func (as *mqttAccountSessionManager) setCachedRetainedMsg(subject string, rm *mqttRetainedMsg, onlyReplace bool, copyBytesToCache bool) { + if as.rmsCache == nil || rm == nil { + return + } rm.expiresFromCache = time.Now().Add(mqttRetainedCacheTTL) if onlyReplace { if _, ok := as.rmsCache.Load(subject); !ok { diff --git a/server/mqtt_ex_test.go b/server/mqtt_ex_test.go new file mode 100644 index 00000000000..44025bef806 --- /dev/null +++ b/server/mqtt_ex_test.go @@ -0,0 +1,424 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_mqtt_tests +// +build !skip_mqtt_tests + +package server + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "strconv" + "testing" + "time" +) + +func TestMQTTExCompliance(t *testing.T) { + mqttPath := os.Getenv("MQTT_CLI") + if mqttPath == "" { + if p, err := exec.LookPath("mqtt"); err == nil { + mqttPath = p + } + } + if mqttPath == "" { + t.Skip(`"mqtt" command is not found in $PATH nor $MQTT_CLI. See https://hivemq.github.io/mqtt-cli/docs/installation/#debian-package for installation instructions`) + } + + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + server_name: mqtt + jetstream { + store_dir = %q + } + mqtt { + listen: 127.0.0.1:-1 + } + `, t.TempDir()))) + s, o := RunServerWithConfig(conf) + defer testMQTTShutdownServer(s) + + cmd := exec.Command(mqttPath, "test", "-V", "3", "-p", strconv.Itoa(o.MQTT.Port)) + + output, err := cmd.CombinedOutput() + t.Log(string(output)) + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + t.Fatalf("mqtt cli exited with error: %v", exitError) + } + } +} + +const ( + KB = 1024 +) + +type mqttBenchMatrix struct { + QOS []int + MessageSize []int + Topics []int + Publishers []int + Subscribers []int +} + +type mqttBenchContext struct { + QOS int + MessageSize int + Topics int + Publishers int + Subscribers int + + Host string + Port int + + // full path to mqtt-test command + testCmdPath string +} + +var mqttBenchDefaultMatrix = mqttBenchMatrix{ + QOS: []int{0, 1, 2}, + MessageSize: []int{100, 1 * KB, 10 * KB}, + Topics: []int{100}, + Publishers: []int{1}, + Subscribers: []int{1}, +} + +type MQTTBenchmarkResult struct { + Ops int `json:"ops"` + NS map[string]time.Duration `json:"ns"` + Bytes int64 `json:"bytes"` +} + +func BenchmarkMQTTEx(b *testing.B) { + bc := mqttNewBenchEx(b) + b.Run("Server", func(b *testing.B) { + b.Cleanup(bc.startServer(b, false)) + bc.runAll(b) + }) + + b.Run("Cluster", func(b *testing.B) { + b.Cleanup(bc.startCluster(b, false)) + bc.runAll(b) + }) + + b.Run("Server-no-RMSCache", func(b *testing.B) { + b.Cleanup(bc.startServer(b, true)) + + bc.benchmarkSubRet(b) + }) + + b.Run("Cluster-no-RMSCache", func(b *testing.B) { + b.Cleanup(bc.startCluster(b, true)) + + bc.benchmarkSubRet(b) + }) +} + +func (bc mqttBenchContext) runAll(b *testing.B) { + bc.benchmarkPub(b) + bc.benchmarkPubRetained(b) + bc.benchmarkPubSub(b) + bc.benchmarkSubRet(b) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkPub(b *testing.B) { + m := mqttBenchDefaultMatrix. + NoSubscribers(). + NoTopics() + + b.Run("PUB", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runCommand(b, "pub", + "--qos", strconv.Itoa(bc.QOS), + "--n", strconv.Itoa(b.N), + "--size", strconv.Itoa(bc.MessageSize), + "--num-publishers", strconv.Itoa(bc.Publishers), + ) + }) + }) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkPubRetained(b *testing.B) { + // This bench is meaningless for QOS0 since the client considers the message + // sent as soon as it's written out. It is also useless for QOS2 since the + // flow takes a lot longer, and the difference of publishing as retained or + // not is lost in the noise. + m := mqttBenchDefaultMatrix. + NoSubscribers(). + NoTopics(). + QOS1Only() + + b.Run("PUBRET", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runCommand(b, "pub", "--retain", + "--qos", strconv.Itoa(bc.QOS), + "--n", strconv.Itoa(b.N), + "--size", strconv.Itoa(bc.MessageSize), + "--num-publishers", strconv.Itoa(bc.Publishers), + ) + }) + }) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkPubSub(b *testing.B) { + // This test uses a single built-in topic, and a built-in publisher, so no + // reason to run it for topics and publishers. + m := mqttBenchDefaultMatrix. + NoTopics(). + NoPublishers() + + b.Run("PUBSUB", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runCommand(b, "pubsub", + "--qos", strconv.Itoa(bc.QOS), + "--n", strconv.Itoa(b.N), + "--size", strconv.Itoa(bc.MessageSize), + "--num-subscribers", strconv.Itoa(bc.Subscribers), + ) + }) + }) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkSubRet(b *testing.B) { + // This test uses a a built-in publisher, and it makes most sense to measure + // the retained message delivery "overhead" on a QoS0 subscription; without + // the extra time involved in actually subscribing. + m := mqttBenchDefaultMatrix. + NoPublishers(). + QOS0Only() + + b.Run("SUBRET", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runCommand(b, "subret", + "--qos", strconv.Itoa(bc.QOS), + "--n", strconv.Itoa(b.N), // number of subscribe requests + "--num-subscribers", strconv.Itoa(bc.Subscribers), + "--num-topics", strconv.Itoa(bc.Topics), + "--size", strconv.Itoa(bc.MessageSize), + ) + }) + }) +} + +func mqttBenchLookupCommand(b *testing.B, name string) string { + b.Helper() + cmd, err := exec.LookPath(name) + if err != nil { + b.Skipf("%q command is not found in $PATH. Please `go install github.com/nats-io/meta-nats/apps/go/mqtt/...@latest` and try again.", name) + } + return cmd +} + +func (bc mqttBenchContext) runCommand(b *testing.B, name string, extraArgs ...string) { + b.Helper() + + args := append([]string{ + name, + "-q", + "--servers", fmt.Sprintf("%s:%d", bc.Host, bc.Port), + }, extraArgs...) + + cmd := exec.Command(bc.testCmdPath, args...) + stdout, err := cmd.StdoutPipe() + if err != nil { + b.Fatalf("Error executing %q: %v", cmd.String(), err) + } + defer stdout.Close() + errbuf := bytes.Buffer{} + cmd.Stderr = &errbuf + if err = cmd.Start(); err != nil { + b.Fatalf("Error executing %q: %v", cmd.String(), err) + } + r := &MQTTBenchmarkResult{} + if err = json.NewDecoder(stdout).Decode(r); err != nil { + b.Fatalf("failed to decode output of %q: %v\n\n%s", cmd.String(), err, errbuf.String()) + } + if err = cmd.Wait(); err != nil { + b.Fatalf("Error executing %q: %v\n\n%s", cmd.String(), err, errbuf.String()) + } + + r.report(b) +} + +func (bc mqttBenchContext) initServer(b *testing.B) { + b.Helper() + bc.runCommand(b, "pubsub", + "--id", "__init__", + "--qos", "0", + "--n", "1", + "--size", "100", + "--num-subscribers", "1") +} + +func (bc *mqttBenchContext) startServer(b *testing.B, disableRMSCache bool) func() { + b.Helper() + b.StopTimer() + prevDisableRMSCache := testDisableRMSCache + testDisableRMSCache = disableRMSCache + o := testMQTTDefaultOptions() + s := testMQTTRunServer(b, o) + + o = s.getOpts() + bc.Host = o.MQTT.Host + bc.Port = o.MQTT.Port + bc.initServer(b) + return func() { + testMQTTShutdownServer(s) + testDisableRMSCache = prevDisableRMSCache + } +} + +func (bc *mqttBenchContext) startCluster(b *testing.B, disableRMSCache bool) func() { + b.Helper() + b.StopTimer() + prevDisableRMSCache := testDisableRMSCache + testDisableRMSCache = disableRMSCache + conf := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + mqtt { + listen: 127.0.0.1:-1 + stream_replicas: 3 + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } + ` + + cl := createJetStreamClusterWithTemplate(b, conf, "MQTT", 3) + o := cl.randomNonLeader().getOpts() + bc.Host = o.MQTT.Host + bc.Port = o.MQTT.Port + bc.initServer(b) + return func() { + cl.shutdown() + testDisableRMSCache = prevDisableRMSCache + } +} + +func mqttBenchWrapForMatrixField( + vFieldPtr *int, + arr []int, + f func(b *testing.B, bc *mqttBenchContext), + namef func(int) string, +) func(b *testing.B, bc *mqttBenchContext) { + if len(arr) == 0 { + return f + } + return func(b *testing.B, bc *mqttBenchContext) { + for _, value := range arr { + *vFieldPtr = value + b.Run(namef(value), func(b *testing.B) { + f(b, bc) + }) + } + } +} + +func (m mqttBenchMatrix) runMatrix(b *testing.B, bc mqttBenchContext, f func(*testing.B, *mqttBenchContext)) { + b.Helper() + f = mqttBenchWrapForMatrixField(&bc.MessageSize, m.MessageSize, f, func(size int) string { + return sizeKB(size) + }) + f = mqttBenchWrapForMatrixField(&bc.Topics, m.Topics, f, func(n int) string { + return fmt.Sprintf("%dtopics", n) + }) + f = mqttBenchWrapForMatrixField(&bc.Publishers, m.Publishers, f, func(n int) string { + return fmt.Sprintf("%dpubc", n) + }) + f = mqttBenchWrapForMatrixField(&bc.Subscribers, m.Subscribers, f, func(n int) string { + return fmt.Sprintf("%dsubc", n) + }) + f = mqttBenchWrapForMatrixField(&bc.QOS, m.QOS, f, func(qos int) string { + return fmt.Sprintf("QOS%d", qos) + }) + b.ResetTimer() + b.StartTimer() + f(b, &bc) +} + +func (m mqttBenchMatrix) NoSubscribers() mqttBenchMatrix { + m.Subscribers = nil + return m +} + +func (m mqttBenchMatrix) NoTopics() mqttBenchMatrix { + m.Topics = nil + return m +} + +func (m mqttBenchMatrix) NoPublishers() mqttBenchMatrix { + m.Publishers = nil + return m +} + +func (m mqttBenchMatrix) QOS0Only() mqttBenchMatrix { + m.QOS = []int{0} + return m +} + +func (m mqttBenchMatrix) QOS1Only() mqttBenchMatrix { + m.QOS = []int{1} + return m +} + +func sizeKB(size int) string { + unit := "B" + N := size + if size >= KB { + unit = "KB" + N = (N + KB/2) / KB + } + return fmt.Sprintf("%d%s", N, unit) +} + +func (r MQTTBenchmarkResult) report(b *testing.B) { + // Disable the default ns metric in favor of custom X_ms/op. + b.ReportMetric(0, "ns/op") + + // Disable MB/s since the github benchmarking action misinterprets the sign + // of the result (treats it as less is better). + b.SetBytes(0) + // b.SetBytes(r.Bytes) + + for unit, ns := range r.NS { + nsOp := float64(ns) / float64(r.Ops) + b.ReportMetric(nsOp/1000000, unit+"_ms/op") + } + + // Diable ReportAllocs() since it confuses the github benchmarking action + // with the noise. + // b.ReportAllocs() +} + +func mqttNewBenchEx(b *testing.B) *mqttBenchContext { + cmd := mqttBenchLookupCommand(b, "mqtt-test") + return &mqttBenchContext{ + testCmdPath: cmd, + } +} From 71c1ca0afde98c6529bad1e33bd841560caa7991 Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Mon, 5 Feb 2024 14:15:57 -0800 Subject: [PATCH 06/21] Add in a JetStream domain elected advisory for when we have a new meta-leader. Signed-off-by: Derek Collison --- server/jetstream_api.go | 3 +++ server/jetstream_cluster.go | 25 +++++++++++++++++++++ server/jetstream_cluster_3_test.go | 35 ++++++++++++++++++++++++++++++ server/jetstream_events.go | 17 ++++++++++++--- 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/server/jetstream_api.go b/server/jetstream_api.go index 14369d6c0e5..a09aa43c9f7 100644 --- a/server/jetstream_api.go +++ b/server/jetstream_api.go @@ -277,6 +277,9 @@ const ( // JSAdvisoryStreamRestoreCompletePre notification that a restore was completed. JSAdvisoryStreamRestoreCompletePre = "$JS.EVENT.ADVISORY.STREAM.RESTORE_COMPLETE" + // JSAdvisoryDomainLeaderElectedPre notification that a jetstream domain has elected a leader. + JSAdvisoryDomainLeaderElected = "$JS.EVENT.ADVISORY.DOMAIN.LEADER_ELECTED" + // JSAdvisoryStreamLeaderElectedPre notification that a replicated stream has elected a leader. JSAdvisoryStreamLeaderElectedPre = "$JS.EVENT.ADVISORY.STREAM.LEADER_ELECTED" diff --git a/server/jetstream_cluster.go b/server/jetstream_cluster.go index e7756cd1144..079741b272b 100644 --- a/server/jetstream_cluster.go +++ b/server/jetstream_cluster.go @@ -5333,6 +5333,30 @@ func (js *jetStream) stopUpdatesSub() { } } +func (s *Server) sendDomainLeaderElectAdvisory() { + js, cc := s.getJetStreamCluster() + if js == nil || cc == nil { + return + } + + js.mu.RLock() + node := cc.meta + js.mu.RUnlock() + + adv := &JSDomainLeaderElectedAdvisory{ + TypedEvent: TypedEvent{ + Type: JSDomainLeaderElectedAdvisoryType, + ID: nuid.Next(), + Time: time.Now().UTC(), + }, + Leader: node.GroupLeader(), + Replicas: s.replicas(node), + Domain: s.getOpts().JetStreamDomain, + } + + s.publishAdvisory(nil, JSAdvisoryDomainLeaderElected, adv) +} + func (js *jetStream) processLeaderChange(isLeader bool) { if js == nil { return @@ -5346,6 +5370,7 @@ func (js *jetStream) processLeaderChange(isLeader bool) { if isLeader { s.Noticef("Self is new JetStream cluster metadata leader") + s.sendDomainLeaderElectAdvisory() } else { var node string if meta := js.getMetaGroup(); meta != nil { diff --git a/server/jetstream_cluster_3_test.go b/server/jetstream_cluster_3_test.go index a269523c1b9..3b93191dd01 100644 --- a/server/jetstream_cluster_3_test.go +++ b/server/jetstream_cluster_3_test.go @@ -5995,3 +5995,38 @@ func TestJetStreamClusterStreamResetPreacks(t *testing.T) { return nil }) } + +func TestJetStreamClusterDomainAdvisory(t *testing.T) { + tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: NGS, store_dir:", 1) + c := createJetStreamCluster(t, tmpl, "R3S", _EMPTY_, 3, 18033, true) + defer c.shutdown() + + // Connect to system account. + nc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo("admin", "s3cr3t!")) + defer nc.Close() + + sub, err := nc.SubscribeSync(JSAdvisoryDomainLeaderElected) + require_NoError(t, err) + + // Ask meta leader to move and make sure we get an advisory. + nc.Request(JSApiLeaderStepDown, nil, time.Second) + c.waitOnLeader() + + checkSubsPending(t, sub, 1) + + m, err := sub.NextMsg(time.Second) + require_NoError(t, err) + + var adv JSDomainLeaderElectedAdvisory + require_NoError(t, json.Unmarshal(m.Data, &adv)) + + ml := c.leader() + js, cc := ml.getJetStreamCluster() + js.mu.RLock() + peer := cc.meta.ID() + js.mu.RUnlock() + + require_Equal(t, adv.Leader, peer) + require_Equal(t, adv.Domain, "NGS") + require_Equal(t, len(adv.Replicas), 3) +} diff --git a/server/jetstream_events.go b/server/jetstream_events.go index 7ea14ef858b..682177b5f6c 100644 --- a/server/jetstream_events.go +++ b/server/jetstream_events.go @@ -193,10 +193,21 @@ const JSRestoreCompleteAdvisoryType = "io.nats.jetstream.advisory.v1.restore_com // Clustering specific. -// JSStreamLeaderElectedAdvisoryType is sent when the system elects a leader for a stream. +// JSClusterLeaderElectedAdvisoryType is sent when the system elects a new meta leader. +const JSDomainLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.domain_leader_elected" + +// JSClusterLeaderElectedAdvisory indicates that a domain has elected a new leader. +type JSDomainLeaderElectedAdvisory struct { + TypedEvent + Leader string `json:"leader"` + Replicas []*PeerInfo `json:"replicas"` + Domain string `json:"domain,omitempty"` +} + +// JSStreamLeaderElectedAdvisoryType is sent when the system elects a new leader for a stream. const JSStreamLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.stream_leader_elected" -// JSStreamLeaderElectedAdvisory indicates that a stream has lost quorum and is stalled. +// JSStreamLeaderElectedAdvisory indicates that a stream has elected a new leader. type JSStreamLeaderElectedAdvisory struct { TypedEvent Account string `json:"account,omitempty"` @@ -222,7 +233,7 @@ type JSStreamQuorumLostAdvisory struct { // JSConsumerLeaderElectedAdvisoryType is sent when the system elects a leader for a consumer. const JSConsumerLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_leader_elected" -// JSConsumerLeaderElectedAdvisory indicates that a stream has lost quorum and is stalled. +// JSConsumerLeaderElectedAdvisory indicates that a consumer has elected a new leader. type JSConsumerLeaderElectedAdvisory struct { TypedEvent Account string `json:"account,omitempty"` From a636f64a75e3e469b6ea1de895203fb566e5d81f Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Wed, 7 Feb 2024 07:51:58 -0800 Subject: [PATCH 07/21] Add in cluster name to advisory Signed-off-by: Derek Collison --- server/jetstream_cluster.go | 1 + server/jetstream_cluster_3_test.go | 1 + server/jetstream_events.go | 1 + 3 files changed, 3 insertions(+) diff --git a/server/jetstream_cluster.go b/server/jetstream_cluster.go index 079741b272b..ee3576fe1b8 100644 --- a/server/jetstream_cluster.go +++ b/server/jetstream_cluster.go @@ -5351,6 +5351,7 @@ func (s *Server) sendDomainLeaderElectAdvisory() { }, Leader: node.GroupLeader(), Replicas: s.replicas(node), + Cluster: s.cachedClusterName(), Domain: s.getOpts().JetStreamDomain, } diff --git a/server/jetstream_cluster_3_test.go b/server/jetstream_cluster_3_test.go index 3b93191dd01..64335c7338e 100644 --- a/server/jetstream_cluster_3_test.go +++ b/server/jetstream_cluster_3_test.go @@ -6028,5 +6028,6 @@ func TestJetStreamClusterDomainAdvisory(t *testing.T) { require_Equal(t, adv.Leader, peer) require_Equal(t, adv.Domain, "NGS") + require_Equal(t, adv.Cluster, "R3S") require_Equal(t, len(adv.Replicas), 3) } diff --git a/server/jetstream_events.go b/server/jetstream_events.go index 682177b5f6c..1852811bb96 100644 --- a/server/jetstream_events.go +++ b/server/jetstream_events.go @@ -201,6 +201,7 @@ type JSDomainLeaderElectedAdvisory struct { TypedEvent Leader string `json:"leader"` Replicas []*PeerInfo `json:"replicas"` + Cluster string `json:"cluster"` Domain string `json:"domain,omitempty"` } From 244f22d9595d7f158591313f6a904699251e5fb8 Mon Sep 17 00:00:00 2001 From: Lev <1187448+levb@users.noreply.github.com> Date: Wed, 7 Feb 2024 12:13:15 -0800 Subject: [PATCH 08/21] FIXED: MQTT retained message consumer creation (#5048) - Cleaned up the retained message consumer name so that it does not cause problems - Per @derekcollison's recommendation, use an ephemeral consumer for retained messages --- server/mqtt.go | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/server/mqtt.go b/server/mqtt.go index f64c160ee58..a1abcb3ef7f 100644 --- a/server/mqtt.go +++ b/server/mqtt.go @@ -1451,21 +1451,12 @@ func (s *Server) mqttCreateAccountSessionManager(acc *Account, quitCh chan struc rmLegacyDurName := mqttRetainedMsgsStreamName + "_" + jsa.id jsa.deleteConsumer(mqttRetainedMsgsStreamName, rmLegacyDurName) - // Using ephemeral consumer is too risky because if this server were to be - // disconnected from the rest for few seconds, then the leader would remove - // the consumer, so even after a reconnect, we would no longer receive - // retained messages. - // - // So we use a durable consumer, and create a new one each time we start. - // The old one should expire and get deleted due to inactivity. The name for - // the durable is $MQTT_rmsgs_{uuid}_{server-name}, the server name is just - // for readability. - rmDurName := mqttRetainedMsgsStreamName + "_" + nuid.Next() + "_" + s.String() - + // Create a new, uniquely names consumer for retained messages for this + // server. The prior one will expire eventually. ccfg := &CreateConsumerRequest{ Stream: mqttRetainedMsgsStreamName, Config: ConsumerConfig{ - Durable: rmDurName, + Name: mqttRetainedMsgsStreamName + "_" + nuid.Next(), FilterSubject: mqttRetainedMsgsStreamSubject + ">", DeliverSubject: rmsubj, ReplayPolicy: ReplayInstant, @@ -1473,7 +1464,7 @@ func (s *Server) mqttCreateAccountSessionManager(acc *Account, quitCh chan struc InactiveThreshold: 5 * time.Minute, }, } - if _, err := jsa.createConsumer(ccfg); err != nil { + if _, err := jsa.createEphemeralConsumer(ccfg); err != nil { return nil, fmt.Errorf("create retained messages consumer for account %q: %v", accName, err) } @@ -1653,17 +1644,26 @@ func (jsa *mqttJSA) sendAck(ackSubject string) { jsa.sendq.push(&mqttJSPubMsg{subj: ackSubject, hdr: -1}) } -func (jsa *mqttJSA) createConsumer(cfg *CreateConsumerRequest) (*JSApiConsumerCreateResponse, error) { +func (jsa *mqttJSA) createEphemeralConsumer(cfg *CreateConsumerRequest) (*JSApiConsumerCreateResponse, error) { cfgb, err := json.Marshal(cfg) if err != nil { return nil, err } - var subj string - if cfg.Config.Durable != _EMPTY_ { - subj = fmt.Sprintf(JSApiDurableCreateT, cfg.Stream, cfg.Config.Durable) - } else { - subj = fmt.Sprintf(JSApiConsumerCreateT, cfg.Stream) + subj := fmt.Sprintf(JSApiConsumerCreateT, cfg.Stream) + ccri, err := jsa.newRequest(mqttJSAConsumerCreate, subj, 0, cfgb) + if err != nil { + return nil, err + } + ccr := ccri.(*JSApiConsumerCreateResponse) + return ccr, ccr.ToError() +} + +func (jsa *mqttJSA) createDurableConsumer(cfg *CreateConsumerRequest) (*JSApiConsumerCreateResponse, error) { + cfgb, err := json.Marshal(cfg) + if err != nil { + return nil, err } + subj := fmt.Sprintf(JSApiDurableCreateT, cfg.Stream, cfg.Config.Durable) ccri, err := jsa.newRequest(mqttJSAConsumerCreate, subj, 0, cfgb) if err != nil { return nil, err @@ -4941,7 +4941,7 @@ func (sess *mqttSession) ensurePubRelConsumerSubscription(c *client) error { if opts.MQTT.ConsumerInactiveThreshold > 0 { ccr.Config.InactiveThreshold = opts.MQTT.ConsumerInactiveThreshold } - if _, err := sess.jsa.createConsumer(ccr); err != nil { + if _, err := sess.jsa.createDurableConsumer(ccr); err != nil { c.Errorf("Unable to add JetStream consumer for PUBREL for client %q: err=%v", id, err) return err } @@ -5047,7 +5047,7 @@ func (sess *mqttSession) processJSConsumer(c *client, subject, sid string, if opts.MQTT.ConsumerInactiveThreshold > 0 { ccr.Config.InactiveThreshold = opts.MQTT.ConsumerInactiveThreshold } - if _, err := sess.jsa.createConsumer(ccr); err != nil { + if _, err := sess.jsa.createDurableConsumer(ccr); err != nil { c.Errorf("Unable to add JetStream consumer for subscription on %q: err=%v", subject, err) return nil, nil, err } From 10e0a010495b6947254b69112e16a54f29f0da46 Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Fri, 9 Feb 2024 15:37:39 -0800 Subject: [PATCH 09/21] [FIXED] KeyValue not found after server restarts (#5054) Do not force sequence mismatches and possible dataloss on first sequence move past a catchup request. When a server restarts and does a stream catchup where the snapshot from the leader is for messages it no longer has, we would create a sequence mismatch reset on the follower. Also on a catchup if the stream state has moved past the updated request range from a follower, meaning first for the new state is past the last of the request, extend and continue catchup. Signed-off-by: Derek Collison --------- Signed-off-by: Derek Collison --- server/jetstream_cluster.go | 25 +++++++-- server/jetstream_cluster_2_test.go | 13 ++--- server/norace_test.go | 85 ++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 11 deletions(-) diff --git a/server/jetstream_cluster.go b/server/jetstream_cluster.go index ee3576fe1b8..532a4eec3da 100644 --- a/server/jetstream_cluster.go +++ b/server/jetstream_cluster.go @@ -8532,10 +8532,13 @@ func (mset *stream) runCatchup(sendSubject string, sreq *streamSyncRequest) { // Reset notion of first if this request wants sequences before our starting sequence // and we would have nothing to send. If we have partial messages still need to send skips for those. + // We will keep sreq's first sequence to not create sequence mismatches on the follower, but we extend the last to our current state. if sreq.FirstSeq < state.FirstSeq && state.FirstSeq > sreq.LastSeq { s.Debugf("Catchup for stream '%s > %s' resetting request first sequence from %d to %d", mset.account(), mset.name(), sreq.FirstSeq, state.FirstSeq) - sreq.FirstSeq = state.FirstSeq + if state.LastSeq > sreq.LastSeq { + sreq.LastSeq = state.LastSeq + } } // Setup sequences to walk through. @@ -8671,10 +8674,22 @@ func (mset *stream) runCatchup(sendSubject string, sreq *streamSyncRequest) { if drOk && dr.First > 0 { sendDR() } - s.Noticef("Catchup for stream '%s > %s' complete", mset.account(), mset.name()) - // EOF - s.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, nil) - return false + // Check for a condition where our state's first is now past the last that we could have sent. + // If so reset last and continue sending. + var state StreamState + mset.mu.RLock() + mset.store.FastState(&state) + mset.mu.RUnlock() + if last < state.FirstSeq { + last = state.LastSeq + } + // Recheck our exit condition. + if seq == last { + s.Noticef("Catchup for stream '%s > %s' complete", mset.account(), mset.name()) + // EOF + s.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, nil) + return false + } } select { case <-remoteQuitCh: diff --git a/server/jetstream_cluster_2_test.go b/server/jetstream_cluster_2_test.go index 7a6f60e6437..8c7e17593fd 100644 --- a/server/jetstream_cluster_2_test.go +++ b/server/jetstream_cluster_2_test.go @@ -6586,14 +6586,14 @@ func TestJetStreamClusterSnapshotBeforePurgeAndCatchup(t *testing.T) { } } - // Send first 100 to everyone. + // Send first 1000 to everyone. send1k() // Now shutdown a non-leader. c.waitOnStreamCurrent(nl, "$G", "TEST") nl.Shutdown() - // Send another 100. + // Send another 1000. send1k() // Force snapshot on the leader. @@ -6606,7 +6606,7 @@ func TestJetStreamClusterSnapshotBeforePurgeAndCatchup(t *testing.T) { err = js.PurgeStream("TEST") require_NoError(t, err) - // Send another 100. + // Send another 1000. send1k() // We want to make sure we do not send unnecessary skip msgs when we know we do not have all of these messages. @@ -6630,10 +6630,11 @@ func TestJetStreamClusterSnapshotBeforePurgeAndCatchup(t *testing.T) { return nil }) - // Make sure we only sent 1 sync catchup msg. + // Make sure we only sent 1002 sync catchup msgs. + // This is for the new messages, the delete range, and the EOF. nmsgs, _, _ := sub.Pending() - if nmsgs != 1 { - t.Fatalf("Expected only 1 sync catchup msg to be sent signaling eof, but got %d", nmsgs) + if nmsgs != 1002 { + t.Fatalf("Expected only 1002 sync catchup msgs to be sent signaling eof, but got %d", nmsgs) } } diff --git a/server/norace_test.go b/server/norace_test.go index 83c4ea8e077..de85203fa7b 100644 --- a/server/norace_test.go +++ b/server/norace_test.go @@ -9514,3 +9514,88 @@ func TestNoRaceJetStreamClusterBadRestartsWithHealthzPolling(t *testing.T) { return nil }) } + +func TestNoRaceJetStreamKVReplaceWithServerRestart(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, _ := jsClientConnect(t, c.randomServer()) + defer nc.Close() + // Shorten wait time for disconnects. + js, err := nc.JetStream(nats.MaxWait(time.Second)) + require_NoError(t, err) + + kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ + Bucket: "TEST", + Replicas: 3, + }) + require_NoError(t, err) + + createData := func(n int) []byte { + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] + } + return b + } + + _, err = kv.Create("foo", createData(160)) + require_NoError(t, err) + + ch := make(chan struct{}) + wg := sync.WaitGroup{} + + // For counting errors that should not happen. + errCh := make(chan error, 1024) + + wg.Add(1) + go func() { + defer wg.Done() + + var lastData []byte + var revision uint64 + + for { + select { + case <-ch: + return + default: + k, err := kv.Get("foo") + if err == nats.ErrKeyNotFound { + errCh <- err + } else if k != nil { + if lastData != nil && k.Revision() == revision && !bytes.Equal(lastData, k.Value()) { + errCh <- fmt.Errorf("data loss [%s][rev:%d] expected:[%q] is:[%q]\n", "foo", revision, lastData, k.Value()) + } + newData := createData(160) + if revision, err = kv.Update("foo", newData, k.Revision()); err == nil { + lastData = newData + } + } + } + } + }() + + // Wait a short bit. + time.Sleep(2 * time.Second) + for _, s := range c.servers { + s.Shutdown() + // Need to leave servers down for awhile to trigger bug properly. + time.Sleep(5 * time.Second) + s = c.restartServer(s) + c.waitOnServerHealthz(s) + } + + // Shutdown the go routine above. + close(ch) + // Wait for it to finish. + wg.Wait() + + if len(errCh) != 0 { + for err := range errCh { + t.Logf("Received err %v during test", err) + } + t.Fatalf("Encountered errors") + } +} From 584312b582bb83a2ca4d03a2bcdbddb5c2acd32b Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Fri, 9 Feb 2024 14:22:42 -0800 Subject: [PATCH 10/21] Add test to check FS State Messages drifting after applying limits Signed-off-by: Waldemar Quevedo --- server/jetstream_cluster_3_test.go | 321 +++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) diff --git a/server/jetstream_cluster_3_test.go b/server/jetstream_cluster_3_test.go index 64335c7338e..4617529661d 100644 --- a/server/jetstream_cluster_3_test.go +++ b/server/jetstream_cluster_3_test.go @@ -6031,3 +6031,324 @@ func TestJetStreamClusterDomainAdvisory(t *testing.T) { require_Equal(t, adv.Cluster, "R3S") require_Equal(t, len(adv.Replicas), 3) } + +func TestJetStreamClusterLimitsBasedStreamFileStoreDesync(t *testing.T) { + conf := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: { + store_dir: '%s', + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + system_account: sys + no_auth_user: js + accounts { + sys { + users = [ + { user: sys, pass: sys } + ] + } + js { + jetstream = { store_max_stream_bytes = 3mb } + users = [ + { user: js, pass: js } + ] + } + }` + c := createJetStreamClusterWithTemplate(t, conf, "limits", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + cnc, cjs := jsClientConnect(t, c.randomServer()) + defer cnc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "LTEST", + Subjects: []string{"messages.*"}, + Replicas: 3, + MaxAge: 10 * time.Minute, + MaxMsgs: 100_000, + }) + require_NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + psub, err := cjs.PullSubscribe("messages.*", "consumer") + require_NoError(t, err) + + var ( + wg sync.WaitGroup + received uint64 + errCh = make(chan error, 100_000) + receivedMap = make(map[string]*nats.Msg) + ) + wg.Add(1) + go func() { + tick := time.NewTicker(20 * time.Millisecond) + for { + select { + case <-ctx.Done(): + wg.Done() + return + case <-tick.C: + msgs, err := psub.Fetch(10, nats.MaxWait(200*time.Millisecond)) + if err != nil { + continue + } + for _, msg := range msgs { + received++ + receivedMap[msg.Subject] = msg + if meta, _ := msg.Metadata(); meta.NumDelivered > 1 { + t.Logf("GOT MSG: %s :: %+v :: %d", msg.Subject, meta, len(msg.Data)) + } + msg.Ack() + } + } + } + }() + + // Send 20_000 msgs at roughly 1 msg per msec + shouldDrop := make(map[string]error) + wg.Add(1) + go func() { + payload := []byte(strings.Repeat("A", 1024)) + tick := time.NewTicker(1 * time.Millisecond) + for i := 1; i < 100_000; { + select { + case <-ctx.Done(): + wg.Done() + return + case <-tick.C: + // This should run into 3MB quota and get errors right away + // before the max msgs limit does. + subject := fmt.Sprintf("messages.%d", i) + _, err := js.Publish(subject, payload, nats.RetryAttempts(0)) + if err != nil { + errCh <- err + } + i++ + + // Any message over this number should not be a success + // since the stream should be full due to the quota. + // Here we capture that the messages have failed to confirm. + if err != nil && i > 1000 { + shouldDrop[subject] = err + } + } + } + }() + + // Collect enough errors to cause things to get out of sync. + var errCount int +Setup: + for { + select { + case err = <-errCh: + errCount++ + if errCount >= 20_000 { + // Stop both producing and consuming. + cancel() + break Setup + } + case <-time.After(5 * time.Second): + t.Fatalf("Timed out waiting for limits error") + } + } + + // Both goroutines should be exiting now.. + wg.Wait() + + // Check messages that ought to have been dropped. + for subject := range receivedMap { + found, ok := shouldDrop[subject] + if ok { + t.Errorf("Should have dropped message published on %q since got error: %v", subject, found) + } + } + + getStreamDetails := func(t *testing.T, srv *Server) *StreamDetail { + t.Helper() + jsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true}) + require_NoError(t, err) + if len(jsz.AccountDetails) > 0 && len(jsz.AccountDetails[0].Streams) > 0 { + details := jsz.AccountDetails[0] + stream := details.Streams[0] + return &stream + } + t.Error("Could not find account details") + return nil + } + checkState := func(t *testing.T) error { + t.Helper() + + leaderSrv := c.streamLeader("js", "LTEST") + streamLeader := getStreamDetails(t, leaderSrv) + // t.Logf("Stream Leader: %+v", streamLeader.State) + errs := make([]error, 0) + for _, srv := range c.servers { + if srv == leaderSrv { + // Skip self + continue + } + stream := getStreamDetails(t, srv) + if stream.State.Msgs != streamLeader.State.Msgs { + err := fmt.Errorf("Leader %v has %d messages, Follower %v has %d messages", + stream.Cluster.Leader, streamLeader.State.Msgs, + srv.Name(), stream.State.Msgs, + ) + errs = append(errs, err) + } + } + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil + } + + // Confirm state of the leader. + leaderSrv := c.streamLeader("js", "LTEST") + streamLeader := getStreamDetails(t, leaderSrv) + if streamLeader.State.Msgs != received { + t.Errorf("Leader %v has %d messages stored but %d messages were received (delta: %d)", + leaderSrv.Name(), streamLeader.State.Msgs, received, received-streamLeader.State.Msgs) + } + cinfo, err := psub.ConsumerInfo() + require_NoError(t, err) + if received != cinfo.Delivered.Consumer { + t.Errorf("Unexpected consumer sequence. Got: %v, expected: %v", + cinfo.Delivered.Consumer, received) + } + + // Check whether there was a drift among the leader and followers. + var ( + lastErr error + attempts int + ) +Check: + for range time.NewTicker(1 * time.Second).C { + lastErr = checkState(t) + if attempts > 5 { + break Check + } + attempts++ + } + + // Read the stream + psub2, err := cjs.PullSubscribe("messages.*", "") + require_NoError(t, err) + +Consume2: + for { + msgs, err := psub2.Fetch(100) + if err != nil { + continue + } + for _, msg := range msgs { + msg.Ack() + + meta, _ := msg.Metadata() + if meta.NumPending == 0 { + break Consume2 + } + } + } + + cinfo2, err := psub2.ConsumerInfo() + require_NoError(t, err) + + a := cinfo.Delivered.Consumer + b := cinfo2.Delivered.Consumer + if a != b { + t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) + } + + // Test is done but replicas were in sync so can stop testing at this point. + if lastErr == nil { + return + } + + // Now we will cause a few step downs while out of sync to get different results. + t.Errorf("Replicas are out of sync:\n%v", lastErr) + + stepDown := func() { + _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "LTEST"), nil, time.Second) + } + // Check StreamInfo in this state then trigger a few step downs. + var prevLeaderMsgs uint64 + leaderSrv = c.streamLeader("js", "LTEST") + sinfo, err := js.StreamInfo("LTEST") + prevLeaderMsgs = sinfo.State.Msgs + for i := 0; i < 10; i++ { + stepDown() + time.Sleep(2 * time.Second) + + leaderSrv = c.streamLeader("js", "LTEST") + sinfo, err = js.StreamInfo("LTEST") + if err != nil { + t.Logf("Error: %v", err) + continue + } + if leaderSrv != nil && sinfo != nil { + t.Logf("When leader is %v, Messages: %d", leaderSrv.Name(), sinfo.State.Msgs) + + // Leave the leader as the replica with less messages that was out of sync. + if prevLeaderMsgs > sinfo.State.Msgs { + break + } + } + } + t.Logf("Changed to use leader %v which has %d messages", leaderSrv.Name(), sinfo.State.Msgs) + + // Read the stream again + psub3, err := cjs.PullSubscribe("messages.*", "") + require_NoError(t, err) + +Consume3: + for { + msgs, err := psub3.Fetch(100) + if err != nil { + continue + } + for _, msg := range msgs { + msg.Ack() + + meta, _ := msg.Metadata() + if meta.NumPending == 0 { + break Consume3 + } + } + } + + cinfo3, err := psub3.ConsumerInfo() + require_NoError(t, err) + + // Compare against consumer that was created before resource limits error + // with one created before the step down. + a = cinfo.Delivered.Consumer + b = cinfo2.Delivered.Consumer + if a != b { + t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) + } + + // Compare against consumer that was created before resource limits error + // with one created AFTER the step down. + a = cinfo.Delivered.Consumer + b = cinfo3.Delivered.Consumer + if a != b { + t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) + } + + // Compare consumers created after the resource limits error. + a = cinfo2.Delivered.Consumer + b = cinfo3.Delivered.Consumer + if a != b { + t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) + } +} From 94fe77d6e8abe2620009a36af8675076f5e99178 Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Fri, 9 Feb 2024 19:14:17 -0800 Subject: [PATCH 11/21] Make jsa limit checks consistent and check both pre proposal for clustered mode. If not clustered only check in processJetStreamMsg. Signed-off-by: Derek Collison --- server/jetstream.go | 25 ++++++++++++++------ server/jetstream_cluster.go | 46 ++++--------------------------------- server/jwt_test.go | 5 ---- server/stream.go | 40 ++++++++++++++------------------ 4 files changed, 40 insertions(+), 76 deletions(-) diff --git a/server/jetstream.go b/server/jetstream.go index cba11d14efa..4ea6e340537 100644 --- a/server/jetstream.go +++ b/server/jetstream.go @@ -2151,6 +2151,10 @@ func (jsa *jsAccount) storageTotals() (uint64, uint64) { } func (jsa *jsAccount) limitsExceeded(storeType StorageType, tierName string, replicas int) (bool, *ApiError) { + return jsa.wouldExceedLimits(storeType, tierName, replicas, _EMPTY_, nil, nil) +} + +func (jsa *jsAccount) wouldExceedLimits(storeType StorageType, tierName string, replicas int, subj string, hdr, msg []byte) (bool, *ApiError) { jsa.usageMu.RLock() defer jsa.usageMu.RUnlock() @@ -2164,24 +2168,31 @@ func (jsa *jsAccount) limitsExceeded(storeType StorageType, tierName string, rep return false, nil } r := int64(replicas) - if r < 1 || tierName == _EMPTY_ { + // Make sure replicas is correct. + if r < 1 { r = 1 } + // This is for limits. If we have no tier, consider all to be flat, vs tiers like R3 where we want to scale limit by replication. + lr := r + if tierName == _EMPTY_ { + lr = 1 + } + // Since tiers are flat we need to scale limit up by replicas when checking. if storeType == MemoryStorage { - totalMem := inUse.total.mem - if selectedLimits.MemoryMaxStreamBytes > 0 && totalMem > selectedLimits.MemoryMaxStreamBytes*r { + totalMem := inUse.total.mem + (int64(memStoreMsgSize(subj, hdr, msg)) * r) + if selectedLimits.MemoryMaxStreamBytes > 0 && totalMem > selectedLimits.MemoryMaxStreamBytes*lr { return true, nil } - if selectedLimits.MaxMemory >= 0 && totalMem > selectedLimits.MaxMemory*r { + if selectedLimits.MaxMemory >= 0 && totalMem > selectedLimits.MaxMemory*lr { return true, nil } } else { - totalStore := inUse.total.store - if selectedLimits.StoreMaxStreamBytes > 0 && totalStore > selectedLimits.StoreMaxStreamBytes*r { + totalStore := inUse.total.store + (int64(fileStoreMsgSize(subj, hdr, msg)) * r) + if selectedLimits.StoreMaxStreamBytes > 0 && totalStore > selectedLimits.StoreMaxStreamBytes*lr { return true, nil } - if selectedLimits.MaxStore >= 0 && totalStore > selectedLimits.MaxStore*r { + if selectedLimits.MaxStore >= 0 && totalStore > selectedLimits.MaxStore*lr { return true, nil } } diff --git a/server/jetstream_cluster.go b/server/jetstream_cluster.go index 532a4eec3da..229d3339cb0 100644 --- a/server/jetstream_cluster.go +++ b/server/jetstream_cluster.go @@ -7494,7 +7494,7 @@ func (mset *stream) processClusteredInboundMsg(subject, reply string, hdr, msg [ mset.mu.RLock() canRespond := !mset.cfg.NoAck && len(reply) > 0 name, stype, store := mset.cfg.Name, mset.cfg.Storage, mset.store - s, js, jsa, st, r, tierName, outq, node := mset.srv, mset.js, mset.jsa, mset.cfg.Storage, int64(mset.cfg.Replicas), mset.tier, mset.outq, mset.node + s, js, jsa, st, r, tierName, outq, node := mset.srv, mset.js, mset.jsa, mset.cfg.Storage, mset.cfg.Replicas, mset.tier, mset.outq, mset.node maxMsgSize, lseq, clfs := int(mset.cfg.MaxMsgSize), mset.lseq, mset.clfs isLeader, isSealed := mset.isLeader(), mset.cfg.Sealed mset.mu.RUnlock() @@ -7532,50 +7532,14 @@ func (mset *stream) processClusteredInboundMsg(subject, reply string, hdr, msg [ } // Check here pre-emptively if we have exceeded our account limits. - var exceeded bool - jsa.usageMu.Lock() - jsaLimits, ok := jsa.limits[tierName] - if !ok { - jsa.usageMu.Unlock() - err := fmt.Errorf("no JetStream resource limits found account: %q", jsa.acc().Name) - s.RateLimitWarnf(err.Error()) - if canRespond { - var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} - resp.Error = NewJSNoLimitsError() - response, _ = json.Marshal(resp) - outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) + if exceeded, err := jsa.wouldExceedLimits(st, tierName, r, subject, hdr, msg); exceeded { + if err == nil { + err = NewJSAccountResourcesExceededError() } - return err - } - t, ok := jsa.usage[tierName] - if !ok { - t = &jsaStorage{} - jsa.usage[tierName] = t - } - // Make sure replicas is correct. - if r < 1 { - r = 1 - } - // This is for limits. If we have no tier, consider all to be flat, vs tiers like R3 where we want to scale limit by replication. - lr := r - if tierName == _EMPTY_ { - lr = 1 - } - // Tiers are flat, meaning the limit for R3 will be 100GB, not 300GB, so compare to total but adjust limits. - if st == MemoryStorage && jsaLimits.MaxMemory > 0 { - exceeded = t.total.mem+(int64(memStoreMsgSize(subject, hdr, msg))*r) > (jsaLimits.MaxMemory * lr) - } else if jsaLimits.MaxStore > 0 { - exceeded = t.total.store+(int64(fileStoreMsgSize(subject, hdr, msg))*r) > (jsaLimits.MaxStore * lr) - } - jsa.usageMu.Unlock() - - // If we have exceeded our account limits go ahead and return. - if exceeded { - err := fmt.Errorf("JetStream resource limits exceeded for account: %q", jsa.acc().Name) s.RateLimitWarnf(err.Error()) if canRespond { var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} - resp.Error = NewJSAccountResourcesExceededError() + resp.Error = err response, _ = json.Marshal(resp) outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) } diff --git a/server/jwt_test.go b/server/jwt_test.go index e5bbd223dcc..28314150f6f 100644 --- a/server/jwt_test.go +++ b/server/jwt_test.go @@ -5467,11 +5467,8 @@ func TestJWTJetStreamTiers(t *testing.T) { require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum consumers limit reached") _, err = js.Publish("testR1-3", msg[:]) - require_NoError(t, err) - _, err = js.Publish("testR1-3", []byte("1")) require_Error(t, err) require_Equal(t, err.Error(), "nats: resource limits exceeded for account") - } func TestJWTJetStreamMaxAckPending(t *testing.T) { @@ -5629,8 +5626,6 @@ func TestJWTJetStreamMaxStreamBytes(t *testing.T) { require_NoError(t, err) // test if we can push more messages into the stream - _, err = js.Publish("baz", msg[:]) - require_NoError(t, err) _, err = js.Publish("baz", msg[:]) // exceeds max stream bytes require_Error(t, err) require_Equal(t, err.Error(), "nats: resource limits exceeded for account") diff --git a/server/stream.go b/server/stream.go index b2bef9a8622..d5b24e0d54d 100644 --- a/server/stream.go +++ b/server/stream.go @@ -4242,10 +4242,10 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte, // Process additional msg headers if still present. var msgId string var rollupSub, rollupAll bool + isClustered := mset.isClustered() if len(hdr) > 0 { outq := mset.outq - isClustered := mset.isClustered() // Certain checks have already been performed if in clustered mode, so only check if not. if !isClustered { @@ -4478,6 +4478,22 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte, } } + // If clustered this was already checked and we do not want to check here and possibly introduce skew. + if !isClustered { + if exceeded, err := jsa.wouldExceedLimits(stype, tierName, mset.cfg.Replicas, subject, hdr, msg); exceeded { + if err == nil { + err = NewJSAccountResourcesExceededError() + } + s.RateLimitWarnf("JetStream resource limits exceeded for account: %q", accName) + if canRespond { + resp.PubAck = &PubAck{Stream: name} + resp.Error = err + response, _ = json.Marshal(resp) + mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) + } + } + } + // Store actual msg. if lseq == 0 && ts == 0 { seq, ts, err = store.StoreMsg(subject, hdr, msg) @@ -4519,28 +4535,6 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte, return err } - if exceeded, apiErr := jsa.limitsExceeded(stype, tierName, mset.cfg.Replicas); exceeded { - s.RateLimitWarnf("JetStream resource limits exceeded for account: %q", accName) - if canRespond { - resp.PubAck = &PubAck{Stream: name} - if apiErr == nil { - resp.Error = NewJSAccountResourcesExceededError() - } else { - resp.Error = apiErr - } - response, _ = json.Marshal(resp) - mset.outq.sendMsg(reply, response) - } - // If we did not succeed put those values back. - var state StreamState - mset.store.FastState(&state) - mset.lseq = state.LastSeq - mset.lmsgId = olmsgId - mset.mu.Unlock() - store.RemoveMsg(seq) - return nil - } - // If we have a msgId make sure to save. if msgId != _EMPTY_ { mset.storeMsgIdLocked(&ddentry{msgId, seq, ts}) From 23b54aaff2bef75c738a424277a7a1d170ab166d Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Sat, 10 Feb 2024 13:59:03 -0800 Subject: [PATCH 12/21] Add workqueue test using discard new policy Signed-off-by: Waldemar Quevedo --- server/jetstream_cluster_3_test.go | 165 +++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/server/jetstream_cluster_3_test.go b/server/jetstream_cluster_3_test.go index 4617529661d..4b2f31227b1 100644 --- a/server/jetstream_cluster_3_test.go +++ b/server/jetstream_cluster_3_test.go @@ -6352,3 +6352,168 @@ Consume3: t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) } } + +func TestJetStreamClusterWorkQueueStreamDiscardNewDesync(t *testing.T) { + t.Run("max msgs", func(t *testing.T) { + testJetStreamClusterWorkQueueStreamDiscardNewDesync(t, &nats.StreamConfig{ + Name: "WQTEST_MM", + Subjects: []string{"messages.*"}, + Replicas: 3, + MaxAge: 10 * time.Minute, + MaxMsgs: 100, + Retention: nats.WorkQueuePolicy, + Discard: nats.DiscardNew, + }) + }) + t.Run("max bytes", func(t *testing.T) { + testJetStreamClusterWorkQueueStreamDiscardNewDesync(t, &nats.StreamConfig{ + Name: "WQTEST_MB", + Subjects: []string{"messages.*"}, + Replicas: 3, + MaxAge: 10 * time.Minute, + MaxBytes: 1 * 1024 * 1024, + Retention: nats.WorkQueuePolicy, + Discard: nats.DiscardNew, + }) + }) +} + +func testJetStreamClusterWorkQueueStreamDiscardNewDesync(t *testing.T, sc *nats.StreamConfig) { + conf := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: { + store_dir: '%s', + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + system_account: sys + no_auth_user: js + accounts { + sys { + users = [ + { user: sys, pass: sys } + ] + } + js { + jetstream = enabled + users = [ + { user: js, pass: js } + ] + } + }` + c := createJetStreamClusterWithTemplate(t, conf, sc.Name, 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + cnc, cjs := jsClientConnect(t, c.randomServer()) + defer cnc.Close() + + _, err := js.AddStream(sc) + require_NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + psub, err := cjs.PullSubscribe("messages.*", "consumer") + require_NoError(t, err) + + stepDown := func() { + _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, sc.Name), nil, time.Second) + } + + // Messages will be produced and consumed in parallel, then once there are + // enough errors a leader election will be triggered. + var ( + wg sync.WaitGroup + received uint64 + errCh = make(chan error, 100_000) + receivedMap = make(map[string]*nats.Msg) + ) + wg.Add(1) + go func() { + tick := time.NewTicker(20 * time.Millisecond) + for { + select { + case <-ctx.Done(): + wg.Done() + return + case <-tick.C: + msgs, err := psub.Fetch(10, nats.MaxWait(200*time.Millisecond)) + if err != nil { + // The consumer will continue to timeout here eventually. + continue + } + for _, msg := range msgs { + received++ + receivedMap[msg.Subject] = msg + msg.Ack() + } + } + } + }() + + shouldDrop := make(map[string]error) + wg.Add(1) + go func() { + payload := []byte(strings.Repeat("A", 1024)) + tick := time.NewTicker(1 * time.Millisecond) + for i := 1; ; i++ { + select { + case <-ctx.Done(): + wg.Done() + return + case <-tick.C: + subject := fmt.Sprintf("messages.%d", i) + _, err := js.Publish(subject, payload, nats.RetryAttempts(0)) + if err != nil { + errCh <- err + } + // Capture the messages that have failed. + if err != nil { + shouldDrop[subject] = err + } + } + } + }() + + // Collect enough errors to cause things to get out of sync. + var errCount int +Setup: + for { + select { + case err = <-errCh: + errCount++ + if errCount%500 == 0 { + stepDown() + } else if errCount >= 2000 { + // Stop both producing and consuming. + cancel() + break Setup + } + case <-time.After(5 * time.Second): + // Unblock the test and continue. + cancel() + break Setup + } + } + + // Both goroutines should be exiting now.. + wg.Wait() + + // Let acks propagate for stream checks. + time.Sleep(250 * time.Millisecond) + + // Check messages that ought to have been dropped. + for subject := range receivedMap { + found, ok := shouldDrop[subject] + if ok { + t.Errorf("Should have dropped message published on %q since got error: %v", subject, found) + } + } +} From 88de9c3615e6fcf818b2770c18dc5ab0ebbc4cc2 Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Sun, 11 Feb 2024 07:59:01 -0800 Subject: [PATCH 13/21] When a stream is replicated and interest policy and has max msgs or max bytes set, we need to do those checks before proposing the message. The reason is that the consumer ack flow will be async state and some replicas might accept the message where others would not based on limits. Signed-off-by: Derek Collison --- server/jetstream_cluster.go | 68 ++++++++++++++++++++++++++++++++++++- server/stream.go | 1 + 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/server/jetstream_cluster.go b/server/jetstream_cluster.go index 229d3339cb0..8f911bcbe32 100644 --- a/server/jetstream_cluster.go +++ b/server/jetstream_cluster.go @@ -2873,7 +2873,18 @@ func (js *jetStream) applyStreamEntries(mset *stream, ce *CommittedEntry, isReco } // Process the actual message here. - if err := mset.processJetStreamMsg(subject, reply, hdr, msg, lseq, ts); err != nil { + err = mset.processJetStreamMsg(subject, reply, hdr, msg, lseq, ts) + + // If we have inflight make sure to clear after processing. + // TODO(dlc) - technically check on inflight != nil could cause datarace. + // But do not want to acquire lock since tracking this will be rare. + if mset.inflight != nil { + mset.clMu.Lock() + delete(mset.inflight, lseq) + mset.clMu.Unlock() + } + + if err != nil { if err == errLastSeqMismatch { var state StreamState mset.store.FastState(&state) @@ -3108,6 +3119,12 @@ func (js *jetStream) processStreamLeaderChange(mset *stream, isLeader bool) { if sa == nil { return } + + // Clear inflight if we have it. + mset.clMu.Lock() + mset.inflight = nil + mset.clMu.Unlock() + js.mu.Lock() s, account, err := js.srv, sa.Client.serviceAccount(), sa.err client, subject, reply := sa.Client, sa.Subject, sa.Reply @@ -7496,7 +7513,14 @@ func (mset *stream) processClusteredInboundMsg(subject, reply string, hdr, msg [ name, stype, store := mset.cfg.Name, mset.cfg.Storage, mset.store s, js, jsa, st, r, tierName, outq, node := mset.srv, mset.js, mset.jsa, mset.cfg.Storage, mset.cfg.Replicas, mset.tier, mset.outq, mset.node maxMsgSize, lseq, clfs := int(mset.cfg.MaxMsgSize), mset.lseq, mset.clfs + interestPolicy, discard, maxMsgs, maxBytes := mset.cfg.Retention != LimitsPolicy, mset.cfg.Discard, mset.cfg.MaxMsgs, mset.cfg.MaxBytes isLeader, isSealed := mset.isLeader(), mset.cfg.Sealed + + // We need to track state to check limits if interest retention and discard new with max msgs or bytes. + var state StreamState + if interestPolicy && discard == DiscardNew && (maxMsgs > 0 || maxBytes > 0) { + mset.store.FastState(&state) + } mset.mu.RUnlock() // This should not happen but possible now that we allow scale up, and scale down where this could trigger. @@ -7618,6 +7642,48 @@ func (mset *stream) processClusteredInboundMsg(subject, reply string, hdr, msg [ lseq, clfs = mset.lseq, mset.clfs mset.clseq = lseq + clfs } + + // Check if we have an interest policy and discard new with max msgs or bytes. + // We need to deny here otherwise it could succeed on some peers and not others + // depending on consumer ack state. So we deny here, if we allow that means we know + // it would succeed on every peer. + if interestPolicy && discard == DiscardNew && (maxMsgs > 0 || maxBytes > 0) { + // Track inflight. + if mset.inflight == nil { + mset.inflight = make(map[uint64]uint64) + } + if mset.cfg.Storage == FileStorage { + mset.inflight[mset.clseq] = fileStoreMsgSize(subject, hdr, msg) + } else { + mset.inflight[mset.clseq] = memStoreMsgSize(subject, hdr, msg) + } + + var err error + if maxMsgs > 0 && state.Msgs+uint64(len(mset.inflight)) > uint64(maxMsgs) { + err = ErrMaxMsgs + } else if maxBytes > 0 { + // TODO(dlc) - Could track this rollup independently. + var bytesPending uint64 + for _, nb := range mset.inflight { + bytesPending += nb + } + if state.Bytes+bytesPending > uint64(maxBytes) { + err = ErrMaxBytes + } + } + if err != nil { + delete(mset.inflight, mset.clseq) + mset.clMu.Unlock() + if canRespond { + var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} + resp.Error = NewJSStreamStoreFailedError(err, Unless(err)) + response, _ = json.Marshal(resp) + outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) + } + return err + } + } + esm := encodeStreamMsgAllowCompress(subject, reply, hdr, msg, mset.clseq, time.Now().UnixNano(), mset.compressOK) mset.clseq++ diff --git a/server/stream.go b/server/stream.go index d5b24e0d54d..58f58f204ff 100644 --- a/server/stream.go +++ b/server/stream.go @@ -283,6 +283,7 @@ type stream struct { clMu sync.Mutex clseq uint64 clfs uint64 + inflight map[uint64]uint64 leader string lqsent time.Time catchups map[string]uint64 From 649c9a12f11741962425b692e9ea75c988fc637b Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Mon, 12 Feb 2024 08:23:44 -0800 Subject: [PATCH 14/21] Update dependencies. Signed-off-by: Derek Collison --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e2810183431..996548dcacb 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module github.com/nats-io/nats-server/v2 go 1.20 require ( - github.com/klauspost/compress v1.17.5 + github.com/klauspost/compress v1.17.6 github.com/minio/highwayhash v1.0.2 github.com/nats-io/jwt/v2 v2.5.3 github.com/nats-io/nats.go v1.32.0 github.com/nats-io/nkeys v0.4.7 github.com/nats-io/nuid v1.0.1 go.uber.org/automaxprocs v1.5.3 - golang.org/x/crypto v0.18.0 - golang.org/x/sys v0.16.0 + golang.org/x/crypto v0.19.0 + golang.org/x/sys v0.17.0 golang.org/x/time v0.5.0 ) diff --git a/go.sum b/go.sum index 09dad5d5193..010a129d499 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/nats-io/jwt/v2 v2.5.3 h1:/9SWvzc6hTfamcgXJ3uYRpgj+QuY2aLNqRiqrKcrpEo= @@ -18,9 +20,13 @@ go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 4cddcace1dbd2d90fc375c2b40daa17930bcfd0c Mon Sep 17 00:00:00 2001 From: Byron Ruth Date: Sun, 11 Feb 2024 08:57:07 -0500 Subject: [PATCH 15/21] Support building for linux/ppc64le arch This request was raised in #555 (2017) and more recently in the nats-docker repo here: https://github.com/nats-io/nats-docker/issues/117. Signed-off-by: Byron Ruth --- .goreleaser.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.goreleaser.yml b/.goreleaser.yml index b1599b43fbe..51a0d21da8b 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -32,6 +32,7 @@ builds: - 386 - mips64le - s390x + - ppc64le goarm: - 6 - 7 From c29cb5b75ca8ee49115d33b0830c0b253e2e59ec Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Tue, 13 Feb 2024 11:47:11 -0800 Subject: [PATCH 16/21] When the server is run with GOMEMLIMIT, make sure to honor that for dynamic memory calculations for JetStream. Signed-off-by: Derek Collison --- server/jetstream.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/jetstream.go b/server/jetstream.go index 4ea6e340537..90bbecd002f 100644 --- a/server/jetstream.go +++ b/server/jetstream.go @@ -23,6 +23,7 @@ import ( "math" "os" "path/filepath" + "runtime/debug" "strconv" "strings" "sync" @@ -2474,6 +2475,11 @@ func (s *Server) dynJetStreamConfig(storeDir string, maxStore, maxMem int64) *Je } else { // Estimate to 75% of total memory if we can determine system memory. if sysMem := sysmem.Memory(); sysMem > 0 { + // Check if we have been limited with GOMEMLIMIT and if lower use that value. + if gml := debug.SetMemoryLimit(-1); gml != math.MaxInt64 && gml < sysMem { + s.Debugf("JetStream detected GOMEMLIMIT of %v", friendlyBytes(gml)) + sysMem = gml + } jsc.MaxMemory = sysMem / 4 * 3 } else { jsc.MaxMemory = JetStreamMaxMemDefault From e0ebe8eec929c75de942f03213d398f70e745002 Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Tue, 13 Feb 2024 11:56:58 -0800 Subject: [PATCH 17/21] Fix cfg access data race Signed-off-by: Derek Collison --- server/jetstream_cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/jetstream_cluster.go b/server/jetstream_cluster.go index 8f911bcbe32..e3c3838d67b 100644 --- a/server/jetstream_cluster.go +++ b/server/jetstream_cluster.go @@ -7877,7 +7877,7 @@ func (mset *stream) processSnapshot(snap *StreamReplicatedState) (e error) { mset.setCLFS(snap.Failed) sreq := mset.calculateSyncRequest(&state, snap) - s, js, subject, n := mset.srv, mset.js, mset.sa.Sync, mset.node + s, js, subject, n, st := mset.srv, mset.js, mset.sa.Sync, mset.node, mset.cfg.Storage qname := fmt.Sprintf("[ACC:%s] stream '%s' snapshot", mset.acc.Name, mset.cfg.Name) mset.mu.Unlock() @@ -7887,7 +7887,7 @@ func (mset *stream) processSnapshot(snap *StreamReplicatedState) (e error) { } // Just return if up to date or already exceeded limits. - if sreq == nil || js.limitsExceeded(mset.cfg.Storage) { + if sreq == nil || js.limitsExceeded(st) { return nil } From a635526fd5263e5a32944d9d9e1a38f19c62cd74 Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Tue, 13 Feb 2024 12:08:23 -0800 Subject: [PATCH 18/21] Fix c.acc access data race Signed-off-by: Derek Collison --- server/client.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/client.go b/server/client.go index 348c0478fec..265b2c73111 100644 --- a/server/client.go +++ b/server/client.go @@ -854,8 +854,12 @@ func (c *client) applyAccountLimits() { } } } + + c.acc.mu.RLock() minLimit(&c.mpay, c.acc.mpay) minLimit(&c.msubs, c.acc.msubs) + c.acc.mu.RUnlock() + s := c.srv opts := s.getOpts() mPay := opts.MaxPayload From a2d3055524b9199aaf4bfcc6ad19aa280b34f365 Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Tue, 13 Feb 2024 13:37:14 -0800 Subject: [PATCH 19/21] Make sure to properly handle filtered purge with a consumer with a wider filtered subject. Signed-off-by: Derek Collison --- server/consumer.go | 32 ++++++++++++++--- server/jetstream_test.go | 77 ++++++++++++++++++++++++++++++++++++++++ server/stream.go | 12 +++++-- 3 files changed, 114 insertions(+), 7 deletions(-) diff --git a/server/consumer.go b/server/consumer.go index a9ad0ebc417..1bf94e0d90c 100644 --- a/server/consumer.go +++ b/server/consumer.go @@ -4901,17 +4901,26 @@ func (o *consumer) hasNoLocalInterest() bool { // This is when the underlying stream has been purged. // sseq is the new first seq for the stream after purge. -// Lock should be held. -func (o *consumer) purge(sseq uint64, slseq uint64) { +// Lock should NOT be held. +func (o *consumer) purge(sseq uint64, slseq uint64, isWider bool) { // Do not update our state unless we know we are the leader. if !o.isLeader() { return } // Signals all have been purged for this consumer. - if sseq == 0 { + if sseq == 0 && !isWider { sseq = slseq + 1 } + var store StreamStore + if isWider { + o.mu.RLock() + if o.mset != nil { + store = o.mset.store + } + o.mu.RUnlock() + } + o.mu.Lock() // Do not go backwards if o.sseq < sseq { @@ -4920,7 +4929,6 @@ func (o *consumer) purge(sseq uint64, slseq uint64) { if o.asflr < sseq { o.asflr = sseq - 1 - // We need to remove those no longer relevant from pending. for seq, p := range o.pending { if seq <= o.asflr { @@ -4934,8 +4942,24 @@ func (o *consumer) purge(sseq uint64, slseq uint64) { delete(o.rdc, seq) // rdq handled below. } + if isWider && store != nil { + // Our filtered subject, which could be all, is wider than the underlying purge. + // We need to check if the pending items left are still valid. + var smv StoreMsg + if _, err := store.LoadMsg(seq, &smv); err == errDeletedMsg || err == ErrStoreMsgNotFound { + if p.Sequence > o.adflr { + o.adflr = p.Sequence + if o.adflr > o.dseq { + o.dseq = o.adflr + } + } + delete(o.pending, seq) + delete(o.rdc, seq) + } + } } } + // This means we can reset everything at this point. if len(o.pending) == 0 { o.pending, o.rdc = nil, nil diff --git a/server/jetstream_test.go b/server/jetstream_test.go index 5006d915161..9e4e4cd35bc 100644 --- a/server/jetstream_test.go +++ b/server/jetstream_test.go @@ -13534,6 +13534,13 @@ func TestJetStreamPurgeAndFilteredConsumers(t *testing.T) { t.Fatalf("Expected NumPending to be 10, got %d", ci.NumPending) } + // Also check unfiltered with interleaving messages. + _, err = js.AddConsumer("S", &nats.ConsumerConfig{ + Durable: "all", + AckPolicy: nats.AckExplicitPolicy, + }) + require_NoError(t, err) + // Now purge only adam. jr, _ := json.Marshal(&JSApiStreamPurgeRequest{Subject: "FOO.adam"}) _, err = nc.Request(fmt.Sprintf(JSApiStreamPurgeT, "S"), jr, time.Second) @@ -13559,6 +13566,12 @@ func TestJetStreamPurgeAndFilteredConsumers(t *testing.T) { if ci.AckFloor.Stream != 20 { t.Fatalf("Expected AckFloor for stream to be 20, got %d", ci.AckFloor.Stream) } + + ci, err = js.ConsumerInfo("S", "all") + require_NoError(t, err) + if ci.NumPending != 10 { + t.Fatalf("Expected NumPending to be 10, got %d", ci.NumPending) + } } // Issue #2662 @@ -22162,3 +22175,67 @@ func TestJetStreamConsumerNakThenAckFloorMove(t *testing.T) { require_Equal(t, ci.AckFloor.Stream, 11) require_Equal(t, ci.NumAckPending, 0) } + +func TestJetStreamSubjectFilteredPurgeClearsPendingAcks(t *testing.T) { + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo", "bar"}, + }) + require_NoError(t, err) + + for i := 0; i < 5; i++ { + js.Publish("foo", []byte("OK")) + js.Publish("bar", []byte("OK")) + } + + // Note that there are no subject filters here, this is deliberate + // as previously the purge with filter code was checking for them. + // We want to prove that unfiltered consumers also get purged. + ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ + Name: "my_consumer", + AckPolicy: nats.AckExplicitPolicy, + MaxAckPending: 10, + }) + require_NoError(t, err) + require_Equal(t, ci.NumPending, 10) + require_Equal(t, ci.NumAckPending, 0) + + sub, err := js.PullSubscribe(">", "", nats.Bind("TEST", "my_consumer")) + require_NoError(t, err) + + msgs, err := sub.Fetch(10) + require_NoError(t, err) + require_Len(t, len(msgs), 10) + + ci, err = js.ConsumerInfo("TEST", "my_consumer") + require_NoError(t, err) + require_Equal(t, ci.NumPending, 0) + require_Equal(t, ci.NumAckPending, 10) + + require_NoError(t, js.PurgeStream("TEST", &nats.StreamPurgeRequest{ + Subject: "foo", + })) + + ci, err = js.ConsumerInfo("TEST", "my_consumer") + require_NoError(t, err) + require_Equal(t, ci.NumPending, 0) + require_Equal(t, ci.NumAckPending, 5) + + for i := 0; i < 5; i++ { + js.Publish("foo", []byte("OK")) + } + msgs, err = sub.Fetch(5) + require_NoError(t, err) + require_Len(t, len(msgs), 5) + + ci, err = js.ConsumerInfo("TEST", "my_consumer") + require_NoError(t, err) + require_Equal(t, ci.NumPending, 0) + require_Equal(t, ci.NumAckPending, 10) +} diff --git a/server/stream.go b/server/stream.go index 58f58f204ff..e4b2381bfda 100644 --- a/server/stream.go +++ b/server/stream.go @@ -1999,12 +1999,13 @@ func (mset *stream) purge(preq *JSApiStreamPurgeRequest) (purged uint64, err err // Purge consumers. // Check for filtered purge. if preq != nil && preq.Subject != _EMPTY_ { - ss := store.FilteredState(state.FirstSeq, preq.Subject) + ss := store.FilteredState(fseq, preq.Subject) fseq = ss.First } mset.clsMu.RLock() for _, o := range mset.cList { + start := fseq o.mu.RLock() // we update consumer sequences if: // no subject was specified, we can purge all consumers sequences @@ -2014,10 +2015,15 @@ func (mset *stream) purge(preq *JSApiStreamPurgeRequest) (purged uint64, err err // or consumer filter subject is subset of purged subject, // but not the other way around. o.isEqualOrSubsetMatch(preq.Subject) + // Check if a consumer has a wider subject space then what we purged + var isWider bool + if !doPurge && preq != nil && o.isFilteredMatch(preq.Subject) { + doPurge, isWider = true, true + start = state.FirstSeq + } o.mu.RUnlock() if doPurge { - o.purge(fseq, lseq) - + o.purge(start, lseq, isWider) } } mset.clsMu.RUnlock() From fcf39ab87c1de736bdd5e99ae11b5d1b2af37797 Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Tue, 13 Feb 2024 15:12:51 -0800 Subject: [PATCH 20/21] Bump Go version to v1.21.7 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9a518edb7be..b687f3adaae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ language: go go: # This should be quoted or use .x, but should not be unquoted. # Remember that a YAML bare float drops trailing zeroes. - - "1.21.6" + - "1.21.7" - "1.20.13" go_import_path: github.com/nats-io/nats-server From 9157fe17642922bc0735b9d29237418371ff4daa Mon Sep 17 00:00:00 2001 From: Lev Brouk Date: Tue, 13 Feb 2024 16:02:03 -0800 Subject: [PATCH 21/21] Fixed flapping TestMQTTDecodeRetainedMessage --- server/mqtt_test.go | 89 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/server/mqtt_test.go b/server/mqtt_test.go index 8fc4b874a35..efbfa2e49aa 100644 --- a/server/mqtt_test.go +++ b/server/mqtt_test.go @@ -7448,6 +7448,95 @@ func TestMQTTJetStreamRepublishAndQoS0Subscribers(t *testing.T) { testMQTTExpectNothing(t, r) } +// Test for auto-cleanup of consumers. +func TestMQTTDecodeRetainedMessage(t *testing.T) { + tdir := t.TempDir() + tmpl := ` + listen: 127.0.0.1:-1 + server_name: mqtt + jetstream { + store_dir = %q + } + + mqtt { + listen: 127.0.0.1:-1 + consumer_inactive_threshold: %q + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } + ` + conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, tdir, "0.2s"))) + s, o := RunServerWithConfig(conf) + defer testMQTTShutdownServer(s) + + // Connect and publish a retained message, this will be in the "newer" form, + // with the metadata in the header. + mc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) + defer mc.Close() + testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) + testMQTTPublish(t, mc, r, 0, false, true, "foo/1", 0, []byte("msg1")) + mc.Close() + + // Store a "legacy", JSON-encoded payload directly into the stream. + nc, js := jsClientConnect(t, s) + defer nc.Close() + + rm := mqttRetainedMsg{ + Origin: "test", + Subject: "foo.2", + Topic: "foo/2", + Flags: mqttPubFlagRetain, + Msg: []byte("msg2"), + } + jsonData, _ := json.Marshal(rm) + _, err := js.PublishMsg(&nats.Msg{ + Subject: mqttRetainedMsgsStreamSubject + rm.Subject, + Data: jsonData, + }) + if err != nil { + t.Fatalf("Error publishing retained message to JS directly: %v", err) + } + + // Restart the server to see that it picks up both retained messages on restart. + s.Shutdown() + s = RunServer(o) + defer testMQTTShutdownServer(s) + + // Connect again, subscribe, and check that we get both messages. + mc, r = testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) + defer mc.Close() + testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) + testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/+", qos: 0}}, []byte{0}) + for i := 0; i < 2; i++ { + b, pl := testMQTTReadPacket(t, r) + if pt := b & mqttPacketMask; pt != mqttPacketPub { + t.Fatalf("Expected PUBLISH packet %x, got %x", mqttPacketPub, pt) + } + _, _, topic := testMQTTGetPubMsgExEx(t, mc, r, mqttPubFlagRetain, pl, "", nil) + if string(topic) != "foo/1" && string(topic) != "foo/2" { + t.Fatalf("Expected foo/1 or foo/2, got %q", topic) + } + } + testMQTTExpectNothing(t, r) + mc.Close() + + // Clear both retained messages. + mc, r = testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) + defer mc.Close() + testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) + testMQTTPublish(t, mc, r, 0, false, true, "foo/1", 0, []byte{}) + testMQTTPublish(t, mc, r, 0, false, true, "foo/2", 0, []byte{}) + mc.Close() + + // Connect again, subscribe, and check that we get nothing. + mc, r = testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) + defer mc.Close() + testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) + testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/+", qos: 0}}, []byte{0}) + testMQTTExpectNothing(t, r) +} + ////////////////////////////////////////////////////////////////////////// // // Benchmarks