diff --git a/.github/workflows/MQTT_test.yaml b/.github/workflows/MQTT_test.yaml index fe73d6e8414..bfbd9210dab 100644 --- a/.github/workflows/MQTT_test.yaml +++ b/.github/workflows/MQTT_test.yaml @@ -1,15 +1,15 @@ -name: MQTT Compliance +name: MQTTEx on: [push, pull_request] +permissions: + pull-requests: write # to comment on PRs + contents: write # to comment on commits (to upload artifacts?) + jobs: test: - strategy: - matrix: - go: ["1.21"] env: GOPATH: /home/runner/work/nats-server GO111MODULE: "on" - runs-on: ubuntu-latest steps: - name: Checkout code @@ -20,18 +20,48 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: ${{matrix.go}} + go-version-file: src/github.com/nats-io/nats-server/go.mod + cache-dependency-path: src/github.com/nats-io/nats-server/go.sum - - name: Install deps - shell: bash --noprofile --norc -x -eo pipefail {0} + - name: Set up testing tools and environment + shell: bash --noprofile --norc -eo pipefail {0} + id: setup run: | wget https://github.com/hivemq/mqtt-cli/releases/download/v4.20.0/mqtt-cli-4.20.0.deb sudo apt install ./mqtt-cli-4.20.0.deb + go install github.com/ConnectEverything/mqtt-test@latest + + # - name: Download benchmark result for ${{ github.base_ref || github.ref_name }} + # uses: dawidd6/action-download-artifact@v2 + # continue-on-error: true + # with: + # path: src/github.com/nats-io/nats-server/bench + # name: bench-output-${{ runner.os }} + # branch: ${{ github.base_ref || github.ref }} - - name: Run tests - shell: bash --noprofile --norc -x -eo pipefail {0} + - name: Run tests and benchmarks + shell: bash --noprofile --norc -eo pipefail {0} run: | - set -e - cd src/github.com/nats-io/nats-server/server - go test -v -vet=off --run=TestMQTTCLICompliance - set +e + cd src/github.com/nats-io/nats-server + go test -v --run='MQTTEx' ./server + # go test --run='-' --count=10 --bench 'MQTT_' ./server | tee output.txt + # go test --run='-' --count=10 --bench 'MQTTEx' --timeout=20m --benchtime=100x ./server | tee -a output.txt + go test --run='-' --count=3 --bench 'MQTTEx' --benchtime=100x ./server + + # - name: Compare benchmarks + # uses: benchmark-action/github-action-benchmark@v1 + # with: + # tool: go + # output-file-path: src/github.com/nats-io/nats-server/output.txt + # github-token: ${{ secrets.GITHUB_TOKEN }} + # alert-threshold: 140% + # comment-on-alert: true + # # fail-on-alert: true + # external-data-json-path: src/github.com/nats-io/nats-server/bench/benchmark-data.json + + # - name: Store benchmark result for ${{ github.ref_name }} + # if: always() + # uses: actions/upload-artifact@v3 + # with: + # path: src/github.com/nats-io/nats-server/bench + # name: bench-output-${{ runner.os }} diff --git a/.goreleaser.yml b/.goreleaser.yml index b1599b43fbe..51a0d21da8b 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -32,6 +32,7 @@ builds: - 386 - mips64le - s390x + - ppc64le goarm: - 6 - 7 diff --git a/.travis.yml b/.travis.yml index 9a518edb7be..b687f3adaae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ language: go go: # This should be quoted or use .x, but should not be unquoted. # Remember that a YAML bare float drops trailing zeroes. - - "1.21.6" + - "1.21.7" - "1.20.13" go_import_path: github.com/nats-io/nats-server diff --git a/go.mod b/go.mod index e2810183431..996548dcacb 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module github.com/nats-io/nats-server/v2 go 1.20 require ( - github.com/klauspost/compress v1.17.5 + github.com/klauspost/compress v1.17.6 github.com/minio/highwayhash v1.0.2 github.com/nats-io/jwt/v2 v2.5.3 github.com/nats-io/nats.go v1.32.0 github.com/nats-io/nkeys v0.4.7 github.com/nats-io/nuid v1.0.1 go.uber.org/automaxprocs v1.5.3 - golang.org/x/crypto v0.18.0 - golang.org/x/sys v0.16.0 + golang.org/x/crypto v0.19.0 + golang.org/x/sys v0.17.0 golang.org/x/time v0.5.0 ) diff --git a/go.sum b/go.sum index 09dad5d5193..010a129d499 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/nats-io/jwt/v2 v2.5.3 h1:/9SWvzc6hTfamcgXJ3uYRpgj+QuY2aLNqRiqrKcrpEo= @@ -18,9 +20,13 @@ go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/server/auth_callout.go b/server/auth_callout.go index 6620a4e5125..318114ebae6 100644 --- a/server/auth_callout.go +++ b/server/auth_callout.go @@ -241,30 +241,35 @@ func (s *Server) processClientOrLeafCallout(c *client, opts *Options) (authorize arc, err := decodeResponse(rc, rmsg, racc) if err != nil { + c.authViolation() respCh <- titleCase(err.Error()) return } vr := jwt.CreateValidationResults() arc.Validate(vr) if len(vr.Issues) > 0 { + c.authViolation() respCh <- fmt.Sprintf("Error validating user JWT: %v", vr.Issues[0]) return } // Make sure that the user is what we requested. if arc.Subject != pub { + c.authViolation() respCh <- fmt.Sprintf("Expected authorized user of %q but got %q on account %q", pub, arc.Subject, racc.Name) return } expiration, allowedConnTypes, err := getExpirationAndAllowedConnections(arc, racc.Name) if err != nil { + c.authViolation() respCh <- titleCase(err.Error()) return } targetAcc, err := assignAccountAndPermissions(arc, racc.Name) if err != nil { + c.authViolation() respCh <- titleCase(err.Error()) return } @@ -280,6 +285,7 @@ func (s *Server) processClientOrLeafCallout(c *client, opts *Options) (authorize // Build internal user and bind to the targeted account. nkuser := buildInternalNkeyUser(arc, allowedConnTypes, targetAcc) if err := c.RegisterNkeyUser(nkuser); err != nil { + c.authViolation() respCh <- fmt.Sprintf("Could not register auth callout user: %v", err) return } diff --git a/server/auth_callout_test.go b/server/auth_callout_test.go index 292da407799..d8634d60cba 100644 --- a/server/auth_callout_test.go +++ b/server/auth_callout_test.go @@ -820,7 +820,6 @@ func testAuthCalloutScopedUser(t *testing.T, allowAnyAccount bool) { // Send the signing key token. This should switch us to the test account, but the user // is signed with the account signing key nc := ac.Connect(nats.UserCredentials(creds), nats.Token(scopedToken)) - require_NoError(t, err) resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) @@ -1713,3 +1712,118 @@ func TestAuthCalloutWSClientTLSCerts(t *testing.T) { require_Equal(t, userInfo.UserID, "dlc") require_Equal(t, userInfo.Account, "FOO") } + +func testConfClientClose(t *testing.T, respondNil bool) { + conf := ` + listen: "127.0.0.1:-1" + server_name: ZZ + accounts { + AUTH { users [ {user: "auth", password: "pwd"} ] } + } + authorization { + timeout: 1s + auth_callout { + # Needs to be a public account nkey, will work for both server config and operator mode. + issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" + account: AUTH + auth_users: [ auth ] + } + } + ` + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + if respondNil { + m.Respond(nil) + } else { + m.Respond(serviceResponse(t, user, si.ID, "", "not today", 0)) + } + } + + at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) + defer at.Cleanup() + + // This one will use callout since not defined in server config. + _, err := at.NewClient(nats.UserInfo("a", "x")) + require_Error(t, err) + require_True(t, strings.Contains(strings.ToLower(err.Error()), nats.AUTHORIZATION_ERR)) +} + +func TestAuthCallout_ClientAuthErrorConf(t *testing.T) { + testConfClientClose(t, true) + testConfClientClose(t, false) +} + +func testAuthCall_ClientAuthErrorOperatorMode(t *testing.T, respondNil bool) { + _, spub := createKey(t) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "$SYS" + sysJwt, err := sysClaim.Encode(oKp) + require_NoError(t, err) + + // AUTH service account. + akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) + require_NoError(t, err) + + apub, err := akp.PublicKey() + require_NoError(t, err) + + // The authorized user for the service. + upub, creds := createAuthServiceUser(t, akp) + defer removeFile(t, creds) + + authClaim := jwt.NewAccountClaims(apub) + authClaim.Name = "AUTH" + authClaim.EnableExternalAuthorization(upub) + authClaim.Authorization.AllowedAccounts.Add("*") + + // the scope for the bearer token which has no permissions + sentinelScope, authKP := newScopedRole(t, "sentinel", nil, nil, false) + sentinelScope.Template.Sub.Deny.Add(">") + sentinelScope.Template.Pub.Deny.Add(">") + sentinelScope.Template.Limits.Subs = 0 + sentinelScope.Template.Payload = 0 + authClaim.SigningKeys.AddScopedSigner(sentinelScope) + + authJwt, err := authClaim.Encode(oKp) + require_NoError(t, err) + + conf := fmt.Sprintf(` + listen: 127.0.0.1:-1 + operator: %s + system_account: %s + resolver: MEM + resolver_preload: { + %s: %s + %s: %s + } + `, ojwt, spub, apub, authJwt, spub, sysJwt) + + handler := func(m *nats.Msg) { + user, si, _, _, _ := decodeAuthRequest(t, m.Data) + if respondNil { + m.Respond(nil) + } else { + m.Respond(serviceResponse(t, user, si.ID, "", "not today", 0)) + } + } + + ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) + defer ac.Cleanup() + + // Bearer token - this has no permissions see sentinelScope + // This is used by all users, and the customization will be in other connect args. + // This needs to also be bound to the authorization account. + creds = createScopedUser(t, akp, authKP) + defer removeFile(t, creds) + + // Send the signing key token. This should switch us to the test account, but the user + // is signed with the account signing key + _, err = ac.NewClient(nats.UserCredentials(creds)) + require_Error(t, err) + require_True(t, strings.Contains(strings.ToLower(err.Error()), nats.AUTHORIZATION_ERR)) +} + +func TestAuthCallout_ClientAuthErrorOperatorMode(t *testing.T) { + testAuthCall_ClientAuthErrorOperatorMode(t, true) + testAuthCall_ClientAuthErrorOperatorMode(t, false) +} diff --git a/server/client.go b/server/client.go index 348c0478fec..265b2c73111 100644 --- a/server/client.go +++ b/server/client.go @@ -854,8 +854,12 @@ func (c *client) applyAccountLimits() { } } } + + c.acc.mu.RLock() minLimit(&c.mpay, c.acc.mpay) minLimit(&c.msubs, c.acc.msubs) + c.acc.mu.RUnlock() + s := c.srv opts := s.getOpts() mPay := opts.MaxPayload diff --git a/server/consumer.go b/server/consumer.go index a9ad0ebc417..1bf94e0d90c 100644 --- a/server/consumer.go +++ b/server/consumer.go @@ -4901,17 +4901,26 @@ func (o *consumer) hasNoLocalInterest() bool { // This is when the underlying stream has been purged. // sseq is the new first seq for the stream after purge. -// Lock should be held. -func (o *consumer) purge(sseq uint64, slseq uint64) { +// Lock should NOT be held. +func (o *consumer) purge(sseq uint64, slseq uint64, isWider bool) { // Do not update our state unless we know we are the leader. if !o.isLeader() { return } // Signals all have been purged for this consumer. - if sseq == 0 { + if sseq == 0 && !isWider { sseq = slseq + 1 } + var store StreamStore + if isWider { + o.mu.RLock() + if o.mset != nil { + store = o.mset.store + } + o.mu.RUnlock() + } + o.mu.Lock() // Do not go backwards if o.sseq < sseq { @@ -4920,7 +4929,6 @@ func (o *consumer) purge(sseq uint64, slseq uint64) { if o.asflr < sseq { o.asflr = sseq - 1 - // We need to remove those no longer relevant from pending. for seq, p := range o.pending { if seq <= o.asflr { @@ -4934,8 +4942,24 @@ func (o *consumer) purge(sseq uint64, slseq uint64) { delete(o.rdc, seq) // rdq handled below. } + if isWider && store != nil { + // Our filtered subject, which could be all, is wider than the underlying purge. + // We need to check if the pending items left are still valid. + var smv StoreMsg + if _, err := store.LoadMsg(seq, &smv); err == errDeletedMsg || err == ErrStoreMsgNotFound { + if p.Sequence > o.adflr { + o.adflr = p.Sequence + if o.adflr > o.dseq { + o.dseq = o.adflr + } + } + delete(o.pending, seq) + delete(o.rdc, seq) + } + } } } + // This means we can reset everything at this point. if len(o.pending) == 0 { o.pending, o.rdc = nil, nil diff --git a/server/filestore.go b/server/filestore.go index 861fe4dc9f3..24e2242a7f9 100644 --- a/server/filestore.go +++ b/server/filestore.go @@ -5553,7 +5553,9 @@ func (mb *msgBlock) loadBlock(buf []byte) ([]byte, error) { buf = buf[:sz] } + <-dios n, err := io.ReadFull(f, buf) + dios <- struct{}{} // On success capture raw bytes size. if err == nil { mb.rbytes = uint64(n) diff --git a/server/jetstream.go b/server/jetstream.go index cba11d14efa..90bbecd002f 100644 --- a/server/jetstream.go +++ b/server/jetstream.go @@ -23,6 +23,7 @@ import ( "math" "os" "path/filepath" + "runtime/debug" "strconv" "strings" "sync" @@ -2151,6 +2152,10 @@ func (jsa *jsAccount) storageTotals() (uint64, uint64) { } func (jsa *jsAccount) limitsExceeded(storeType StorageType, tierName string, replicas int) (bool, *ApiError) { + return jsa.wouldExceedLimits(storeType, tierName, replicas, _EMPTY_, nil, nil) +} + +func (jsa *jsAccount) wouldExceedLimits(storeType StorageType, tierName string, replicas int, subj string, hdr, msg []byte) (bool, *ApiError) { jsa.usageMu.RLock() defer jsa.usageMu.RUnlock() @@ -2164,24 +2169,31 @@ func (jsa *jsAccount) limitsExceeded(storeType StorageType, tierName string, rep return false, nil } r := int64(replicas) - if r < 1 || tierName == _EMPTY_ { + // Make sure replicas is correct. + if r < 1 { r = 1 } + // This is for limits. If we have no tier, consider all to be flat, vs tiers like R3 where we want to scale limit by replication. + lr := r + if tierName == _EMPTY_ { + lr = 1 + } + // Since tiers are flat we need to scale limit up by replicas when checking. if storeType == MemoryStorage { - totalMem := inUse.total.mem - if selectedLimits.MemoryMaxStreamBytes > 0 && totalMem > selectedLimits.MemoryMaxStreamBytes*r { + totalMem := inUse.total.mem + (int64(memStoreMsgSize(subj, hdr, msg)) * r) + if selectedLimits.MemoryMaxStreamBytes > 0 && totalMem > selectedLimits.MemoryMaxStreamBytes*lr { return true, nil } - if selectedLimits.MaxMemory >= 0 && totalMem > selectedLimits.MaxMemory*r { + if selectedLimits.MaxMemory >= 0 && totalMem > selectedLimits.MaxMemory*lr { return true, nil } } else { - totalStore := inUse.total.store - if selectedLimits.StoreMaxStreamBytes > 0 && totalStore > selectedLimits.StoreMaxStreamBytes*r { + totalStore := inUse.total.store + (int64(fileStoreMsgSize(subj, hdr, msg)) * r) + if selectedLimits.StoreMaxStreamBytes > 0 && totalStore > selectedLimits.StoreMaxStreamBytes*lr { return true, nil } - if selectedLimits.MaxStore >= 0 && totalStore > selectedLimits.MaxStore*r { + if selectedLimits.MaxStore >= 0 && totalStore > selectedLimits.MaxStore*lr { return true, nil } } @@ -2463,6 +2475,11 @@ func (s *Server) dynJetStreamConfig(storeDir string, maxStore, maxMem int64) *Je } else { // Estimate to 75% of total memory if we can determine system memory. if sysMem := sysmem.Memory(); sysMem > 0 { + // Check if we have been limited with GOMEMLIMIT and if lower use that value. + if gml := debug.SetMemoryLimit(-1); gml != math.MaxInt64 && gml < sysMem { + s.Debugf("JetStream detected GOMEMLIMIT of %v", friendlyBytes(gml)) + sysMem = gml + } jsc.MaxMemory = sysMem / 4 * 3 } else { jsc.MaxMemory = JetStreamMaxMemDefault diff --git a/server/jetstream_api.go b/server/jetstream_api.go index 14369d6c0e5..a09aa43c9f7 100644 --- a/server/jetstream_api.go +++ b/server/jetstream_api.go @@ -277,6 +277,9 @@ const ( // JSAdvisoryStreamRestoreCompletePre notification that a restore was completed. JSAdvisoryStreamRestoreCompletePre = "$JS.EVENT.ADVISORY.STREAM.RESTORE_COMPLETE" + // JSAdvisoryDomainLeaderElectedPre notification that a jetstream domain has elected a leader. + JSAdvisoryDomainLeaderElected = "$JS.EVENT.ADVISORY.DOMAIN.LEADER_ELECTED" + // JSAdvisoryStreamLeaderElectedPre notification that a replicated stream has elected a leader. JSAdvisoryStreamLeaderElectedPre = "$JS.EVENT.ADVISORY.STREAM.LEADER_ELECTED" diff --git a/server/jetstream_cluster.go b/server/jetstream_cluster.go index e7756cd1144..e3c3838d67b 100644 --- a/server/jetstream_cluster.go +++ b/server/jetstream_cluster.go @@ -2873,7 +2873,18 @@ func (js *jetStream) applyStreamEntries(mset *stream, ce *CommittedEntry, isReco } // Process the actual message here. - if err := mset.processJetStreamMsg(subject, reply, hdr, msg, lseq, ts); err != nil { + err = mset.processJetStreamMsg(subject, reply, hdr, msg, lseq, ts) + + // If we have inflight make sure to clear after processing. + // TODO(dlc) - technically check on inflight != nil could cause datarace. + // But do not want to acquire lock since tracking this will be rare. + if mset.inflight != nil { + mset.clMu.Lock() + delete(mset.inflight, lseq) + mset.clMu.Unlock() + } + + if err != nil { if err == errLastSeqMismatch { var state StreamState mset.store.FastState(&state) @@ -3108,6 +3119,12 @@ func (js *jetStream) processStreamLeaderChange(mset *stream, isLeader bool) { if sa == nil { return } + + // Clear inflight if we have it. + mset.clMu.Lock() + mset.inflight = nil + mset.clMu.Unlock() + js.mu.Lock() s, account, err := js.srv, sa.Client.serviceAccount(), sa.err client, subject, reply := sa.Client, sa.Subject, sa.Reply @@ -5333,6 +5350,31 @@ func (js *jetStream) stopUpdatesSub() { } } +func (s *Server) sendDomainLeaderElectAdvisory() { + js, cc := s.getJetStreamCluster() + if js == nil || cc == nil { + return + } + + js.mu.RLock() + node := cc.meta + js.mu.RUnlock() + + adv := &JSDomainLeaderElectedAdvisory{ + TypedEvent: TypedEvent{ + Type: JSDomainLeaderElectedAdvisoryType, + ID: nuid.Next(), + Time: time.Now().UTC(), + }, + Leader: node.GroupLeader(), + Replicas: s.replicas(node), + Cluster: s.cachedClusterName(), + Domain: s.getOpts().JetStreamDomain, + } + + s.publishAdvisory(nil, JSAdvisoryDomainLeaderElected, adv) +} + func (js *jetStream) processLeaderChange(isLeader bool) { if js == nil { return @@ -5346,6 +5388,7 @@ func (js *jetStream) processLeaderChange(isLeader bool) { if isLeader { s.Noticef("Self is new JetStream cluster metadata leader") + s.sendDomainLeaderElectAdvisory() } else { var node string if meta := js.getMetaGroup(); meta != nil { @@ -7468,9 +7511,16 @@ func (mset *stream) processClusteredInboundMsg(subject, reply string, hdr, msg [ mset.mu.RLock() canRespond := !mset.cfg.NoAck && len(reply) > 0 name, stype, store := mset.cfg.Name, mset.cfg.Storage, mset.store - s, js, jsa, st, r, tierName, outq, node := mset.srv, mset.js, mset.jsa, mset.cfg.Storage, int64(mset.cfg.Replicas), mset.tier, mset.outq, mset.node + s, js, jsa, st, r, tierName, outq, node := mset.srv, mset.js, mset.jsa, mset.cfg.Storage, mset.cfg.Replicas, mset.tier, mset.outq, mset.node maxMsgSize, lseq, clfs := int(mset.cfg.MaxMsgSize), mset.lseq, mset.clfs + interestPolicy, discard, maxMsgs, maxBytes := mset.cfg.Retention != LimitsPolicy, mset.cfg.Discard, mset.cfg.MaxMsgs, mset.cfg.MaxBytes isLeader, isSealed := mset.isLeader(), mset.cfg.Sealed + + // We need to track state to check limits if interest retention and discard new with max msgs or bytes. + var state StreamState + if interestPolicy && discard == DiscardNew && (maxMsgs > 0 || maxBytes > 0) { + mset.store.FastState(&state) + } mset.mu.RUnlock() // This should not happen but possible now that we allow scale up, and scale down where this could trigger. @@ -7506,50 +7556,14 @@ func (mset *stream) processClusteredInboundMsg(subject, reply string, hdr, msg [ } // Check here pre-emptively if we have exceeded our account limits. - var exceeded bool - jsa.usageMu.Lock() - jsaLimits, ok := jsa.limits[tierName] - if !ok { - jsa.usageMu.Unlock() - err := fmt.Errorf("no JetStream resource limits found account: %q", jsa.acc().Name) - s.RateLimitWarnf(err.Error()) - if canRespond { - var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} - resp.Error = NewJSNoLimitsError() - response, _ = json.Marshal(resp) - outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) + if exceeded, err := jsa.wouldExceedLimits(st, tierName, r, subject, hdr, msg); exceeded { + if err == nil { + err = NewJSAccountResourcesExceededError() } - return err - } - t, ok := jsa.usage[tierName] - if !ok { - t = &jsaStorage{} - jsa.usage[tierName] = t - } - // Make sure replicas is correct. - if r < 1 { - r = 1 - } - // This is for limits. If we have no tier, consider all to be flat, vs tiers like R3 where we want to scale limit by replication. - lr := r - if tierName == _EMPTY_ { - lr = 1 - } - // Tiers are flat, meaning the limit for R3 will be 100GB, not 300GB, so compare to total but adjust limits. - if st == MemoryStorage && jsaLimits.MaxMemory > 0 { - exceeded = t.total.mem+(int64(memStoreMsgSize(subject, hdr, msg))*r) > (jsaLimits.MaxMemory * lr) - } else if jsaLimits.MaxStore > 0 { - exceeded = t.total.store+(int64(fileStoreMsgSize(subject, hdr, msg))*r) > (jsaLimits.MaxStore * lr) - } - jsa.usageMu.Unlock() - - // If we have exceeded our account limits go ahead and return. - if exceeded { - err := fmt.Errorf("JetStream resource limits exceeded for account: %q", jsa.acc().Name) s.RateLimitWarnf(err.Error()) if canRespond { var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} - resp.Error = NewJSAccountResourcesExceededError() + resp.Error = err response, _ = json.Marshal(resp) outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) } @@ -7628,6 +7642,48 @@ func (mset *stream) processClusteredInboundMsg(subject, reply string, hdr, msg [ lseq, clfs = mset.lseq, mset.clfs mset.clseq = lseq + clfs } + + // Check if we have an interest policy and discard new with max msgs or bytes. + // We need to deny here otherwise it could succeed on some peers and not others + // depending on consumer ack state. So we deny here, if we allow that means we know + // it would succeed on every peer. + if interestPolicy && discard == DiscardNew && (maxMsgs > 0 || maxBytes > 0) { + // Track inflight. + if mset.inflight == nil { + mset.inflight = make(map[uint64]uint64) + } + if mset.cfg.Storage == FileStorage { + mset.inflight[mset.clseq] = fileStoreMsgSize(subject, hdr, msg) + } else { + mset.inflight[mset.clseq] = memStoreMsgSize(subject, hdr, msg) + } + + var err error + if maxMsgs > 0 && state.Msgs+uint64(len(mset.inflight)) > uint64(maxMsgs) { + err = ErrMaxMsgs + } else if maxBytes > 0 { + // TODO(dlc) - Could track this rollup independently. + var bytesPending uint64 + for _, nb := range mset.inflight { + bytesPending += nb + } + if state.Bytes+bytesPending > uint64(maxBytes) { + err = ErrMaxBytes + } + } + if err != nil { + delete(mset.inflight, mset.clseq) + mset.clMu.Unlock() + if canRespond { + var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} + resp.Error = NewJSStreamStoreFailedError(err, Unless(err)) + response, _ = json.Marshal(resp) + outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) + } + return err + } + } + esm := encodeStreamMsgAllowCompress(subject, reply, hdr, msg, mset.clseq, time.Now().UnixNano(), mset.compressOK) mset.clseq++ @@ -7821,7 +7877,7 @@ func (mset *stream) processSnapshot(snap *StreamReplicatedState) (e error) { mset.setCLFS(snap.Failed) sreq := mset.calculateSyncRequest(&state, snap) - s, js, subject, n := mset.srv, mset.js, mset.sa.Sync, mset.node + s, js, subject, n, st := mset.srv, mset.js, mset.sa.Sync, mset.node, mset.cfg.Storage qname := fmt.Sprintf("[ACC:%s] stream '%s' snapshot", mset.acc.Name, mset.cfg.Name) mset.mu.Unlock() @@ -7831,7 +7887,7 @@ func (mset *stream) processSnapshot(snap *StreamReplicatedState) (e error) { } // Just return if up to date or already exceeded limits. - if sreq == nil || js.limitsExceeded(mset.cfg.Storage) { + if sreq == nil || js.limitsExceeded(st) { return nil } @@ -8506,10 +8562,13 @@ func (mset *stream) runCatchup(sendSubject string, sreq *streamSyncRequest) { // Reset notion of first if this request wants sequences before our starting sequence // and we would have nothing to send. If we have partial messages still need to send skips for those. + // We will keep sreq's first sequence to not create sequence mismatches on the follower, but we extend the last to our current state. if sreq.FirstSeq < state.FirstSeq && state.FirstSeq > sreq.LastSeq { s.Debugf("Catchup for stream '%s > %s' resetting request first sequence from %d to %d", mset.account(), mset.name(), sreq.FirstSeq, state.FirstSeq) - sreq.FirstSeq = state.FirstSeq + if state.LastSeq > sreq.LastSeq { + sreq.LastSeq = state.LastSeq + } } // Setup sequences to walk through. @@ -8645,10 +8704,22 @@ func (mset *stream) runCatchup(sendSubject string, sreq *streamSyncRequest) { if drOk && dr.First > 0 { sendDR() } - s.Noticef("Catchup for stream '%s > %s' complete", mset.account(), mset.name()) - // EOF - s.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, nil) - return false + // Check for a condition where our state's first is now past the last that we could have sent. + // If so reset last and continue sending. + var state StreamState + mset.mu.RLock() + mset.store.FastState(&state) + mset.mu.RUnlock() + if last < state.FirstSeq { + last = state.LastSeq + } + // Recheck our exit condition. + if seq == last { + s.Noticef("Catchup for stream '%s > %s' complete", mset.account(), mset.name()) + // EOF + s.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, nil) + return false + } } select { case <-remoteQuitCh: diff --git a/server/jetstream_cluster_2_test.go b/server/jetstream_cluster_2_test.go index 7a6f60e6437..8c7e17593fd 100644 --- a/server/jetstream_cluster_2_test.go +++ b/server/jetstream_cluster_2_test.go @@ -6586,14 +6586,14 @@ func TestJetStreamClusterSnapshotBeforePurgeAndCatchup(t *testing.T) { } } - // Send first 100 to everyone. + // Send first 1000 to everyone. send1k() // Now shutdown a non-leader. c.waitOnStreamCurrent(nl, "$G", "TEST") nl.Shutdown() - // Send another 100. + // Send another 1000. send1k() // Force snapshot on the leader. @@ -6606,7 +6606,7 @@ func TestJetStreamClusterSnapshotBeforePurgeAndCatchup(t *testing.T) { err = js.PurgeStream("TEST") require_NoError(t, err) - // Send another 100. + // Send another 1000. send1k() // We want to make sure we do not send unnecessary skip msgs when we know we do not have all of these messages. @@ -6630,10 +6630,11 @@ func TestJetStreamClusterSnapshotBeforePurgeAndCatchup(t *testing.T) { return nil }) - // Make sure we only sent 1 sync catchup msg. + // Make sure we only sent 1002 sync catchup msgs. + // This is for the new messages, the delete range, and the EOF. nmsgs, _, _ := sub.Pending() - if nmsgs != 1 { - t.Fatalf("Expected only 1 sync catchup msg to be sent signaling eof, but got %d", nmsgs) + if nmsgs != 1002 { + t.Fatalf("Expected only 1002 sync catchup msgs to be sent signaling eof, but got %d", nmsgs) } } diff --git a/server/jetstream_cluster_3_test.go b/server/jetstream_cluster_3_test.go index a269523c1b9..4b2f31227b1 100644 --- a/server/jetstream_cluster_3_test.go +++ b/server/jetstream_cluster_3_test.go @@ -5995,3 +5995,525 @@ func TestJetStreamClusterStreamResetPreacks(t *testing.T) { return nil }) } + +func TestJetStreamClusterDomainAdvisory(t *testing.T) { + tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: NGS, store_dir:", 1) + c := createJetStreamCluster(t, tmpl, "R3S", _EMPTY_, 3, 18033, true) + defer c.shutdown() + + // Connect to system account. + nc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo("admin", "s3cr3t!")) + defer nc.Close() + + sub, err := nc.SubscribeSync(JSAdvisoryDomainLeaderElected) + require_NoError(t, err) + + // Ask meta leader to move and make sure we get an advisory. + nc.Request(JSApiLeaderStepDown, nil, time.Second) + c.waitOnLeader() + + checkSubsPending(t, sub, 1) + + m, err := sub.NextMsg(time.Second) + require_NoError(t, err) + + var adv JSDomainLeaderElectedAdvisory + require_NoError(t, json.Unmarshal(m.Data, &adv)) + + ml := c.leader() + js, cc := ml.getJetStreamCluster() + js.mu.RLock() + peer := cc.meta.ID() + js.mu.RUnlock() + + require_Equal(t, adv.Leader, peer) + require_Equal(t, adv.Domain, "NGS") + require_Equal(t, adv.Cluster, "R3S") + require_Equal(t, len(adv.Replicas), 3) +} + +func TestJetStreamClusterLimitsBasedStreamFileStoreDesync(t *testing.T) { + conf := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: { + store_dir: '%s', + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + system_account: sys + no_auth_user: js + accounts { + sys { + users = [ + { user: sys, pass: sys } + ] + } + js { + jetstream = { store_max_stream_bytes = 3mb } + users = [ + { user: js, pass: js } + ] + } + }` + c := createJetStreamClusterWithTemplate(t, conf, "limits", 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + cnc, cjs := jsClientConnect(t, c.randomServer()) + defer cnc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "LTEST", + Subjects: []string{"messages.*"}, + Replicas: 3, + MaxAge: 10 * time.Minute, + MaxMsgs: 100_000, + }) + require_NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + psub, err := cjs.PullSubscribe("messages.*", "consumer") + require_NoError(t, err) + + var ( + wg sync.WaitGroup + received uint64 + errCh = make(chan error, 100_000) + receivedMap = make(map[string]*nats.Msg) + ) + wg.Add(1) + go func() { + tick := time.NewTicker(20 * time.Millisecond) + for { + select { + case <-ctx.Done(): + wg.Done() + return + case <-tick.C: + msgs, err := psub.Fetch(10, nats.MaxWait(200*time.Millisecond)) + if err != nil { + continue + } + for _, msg := range msgs { + received++ + receivedMap[msg.Subject] = msg + if meta, _ := msg.Metadata(); meta.NumDelivered > 1 { + t.Logf("GOT MSG: %s :: %+v :: %d", msg.Subject, meta, len(msg.Data)) + } + msg.Ack() + } + } + } + }() + + // Send 20_000 msgs at roughly 1 msg per msec + shouldDrop := make(map[string]error) + wg.Add(1) + go func() { + payload := []byte(strings.Repeat("A", 1024)) + tick := time.NewTicker(1 * time.Millisecond) + for i := 1; i < 100_000; { + select { + case <-ctx.Done(): + wg.Done() + return + case <-tick.C: + // This should run into 3MB quota and get errors right away + // before the max msgs limit does. + subject := fmt.Sprintf("messages.%d", i) + _, err := js.Publish(subject, payload, nats.RetryAttempts(0)) + if err != nil { + errCh <- err + } + i++ + + // Any message over this number should not be a success + // since the stream should be full due to the quota. + // Here we capture that the messages have failed to confirm. + if err != nil && i > 1000 { + shouldDrop[subject] = err + } + } + } + }() + + // Collect enough errors to cause things to get out of sync. + var errCount int +Setup: + for { + select { + case err = <-errCh: + errCount++ + if errCount >= 20_000 { + // Stop both producing and consuming. + cancel() + break Setup + } + case <-time.After(5 * time.Second): + t.Fatalf("Timed out waiting for limits error") + } + } + + // Both goroutines should be exiting now.. + wg.Wait() + + // Check messages that ought to have been dropped. + for subject := range receivedMap { + found, ok := shouldDrop[subject] + if ok { + t.Errorf("Should have dropped message published on %q since got error: %v", subject, found) + } + } + + getStreamDetails := func(t *testing.T, srv *Server) *StreamDetail { + t.Helper() + jsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true}) + require_NoError(t, err) + if len(jsz.AccountDetails) > 0 && len(jsz.AccountDetails[0].Streams) > 0 { + details := jsz.AccountDetails[0] + stream := details.Streams[0] + return &stream + } + t.Error("Could not find account details") + return nil + } + checkState := func(t *testing.T) error { + t.Helper() + + leaderSrv := c.streamLeader("js", "LTEST") + streamLeader := getStreamDetails(t, leaderSrv) + // t.Logf("Stream Leader: %+v", streamLeader.State) + errs := make([]error, 0) + for _, srv := range c.servers { + if srv == leaderSrv { + // Skip self + continue + } + stream := getStreamDetails(t, srv) + if stream.State.Msgs != streamLeader.State.Msgs { + err := fmt.Errorf("Leader %v has %d messages, Follower %v has %d messages", + stream.Cluster.Leader, streamLeader.State.Msgs, + srv.Name(), stream.State.Msgs, + ) + errs = append(errs, err) + } + } + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil + } + + // Confirm state of the leader. + leaderSrv := c.streamLeader("js", "LTEST") + streamLeader := getStreamDetails(t, leaderSrv) + if streamLeader.State.Msgs != received { + t.Errorf("Leader %v has %d messages stored but %d messages were received (delta: %d)", + leaderSrv.Name(), streamLeader.State.Msgs, received, received-streamLeader.State.Msgs) + } + cinfo, err := psub.ConsumerInfo() + require_NoError(t, err) + if received != cinfo.Delivered.Consumer { + t.Errorf("Unexpected consumer sequence. Got: %v, expected: %v", + cinfo.Delivered.Consumer, received) + } + + // Check whether there was a drift among the leader and followers. + var ( + lastErr error + attempts int + ) +Check: + for range time.NewTicker(1 * time.Second).C { + lastErr = checkState(t) + if attempts > 5 { + break Check + } + attempts++ + } + + // Read the stream + psub2, err := cjs.PullSubscribe("messages.*", "") + require_NoError(t, err) + +Consume2: + for { + msgs, err := psub2.Fetch(100) + if err != nil { + continue + } + for _, msg := range msgs { + msg.Ack() + + meta, _ := msg.Metadata() + if meta.NumPending == 0 { + break Consume2 + } + } + } + + cinfo2, err := psub2.ConsumerInfo() + require_NoError(t, err) + + a := cinfo.Delivered.Consumer + b := cinfo2.Delivered.Consumer + if a != b { + t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) + } + + // Test is done but replicas were in sync so can stop testing at this point. + if lastErr == nil { + return + } + + // Now we will cause a few step downs while out of sync to get different results. + t.Errorf("Replicas are out of sync:\n%v", lastErr) + + stepDown := func() { + _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "LTEST"), nil, time.Second) + } + // Check StreamInfo in this state then trigger a few step downs. + var prevLeaderMsgs uint64 + leaderSrv = c.streamLeader("js", "LTEST") + sinfo, err := js.StreamInfo("LTEST") + prevLeaderMsgs = sinfo.State.Msgs + for i := 0; i < 10; i++ { + stepDown() + time.Sleep(2 * time.Second) + + leaderSrv = c.streamLeader("js", "LTEST") + sinfo, err = js.StreamInfo("LTEST") + if err != nil { + t.Logf("Error: %v", err) + continue + } + if leaderSrv != nil && sinfo != nil { + t.Logf("When leader is %v, Messages: %d", leaderSrv.Name(), sinfo.State.Msgs) + + // Leave the leader as the replica with less messages that was out of sync. + if prevLeaderMsgs > sinfo.State.Msgs { + break + } + } + } + t.Logf("Changed to use leader %v which has %d messages", leaderSrv.Name(), sinfo.State.Msgs) + + // Read the stream again + psub3, err := cjs.PullSubscribe("messages.*", "") + require_NoError(t, err) + +Consume3: + for { + msgs, err := psub3.Fetch(100) + if err != nil { + continue + } + for _, msg := range msgs { + msg.Ack() + + meta, _ := msg.Metadata() + if meta.NumPending == 0 { + break Consume3 + } + } + } + + cinfo3, err := psub3.ConsumerInfo() + require_NoError(t, err) + + // Compare against consumer that was created before resource limits error + // with one created before the step down. + a = cinfo.Delivered.Consumer + b = cinfo2.Delivered.Consumer + if a != b { + t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) + } + + // Compare against consumer that was created before resource limits error + // with one created AFTER the step down. + a = cinfo.Delivered.Consumer + b = cinfo3.Delivered.Consumer + if a != b { + t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) + } + + // Compare consumers created after the resource limits error. + a = cinfo2.Delivered.Consumer + b = cinfo3.Delivered.Consumer + if a != b { + t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) + } +} + +func TestJetStreamClusterWorkQueueStreamDiscardNewDesync(t *testing.T) { + t.Run("max msgs", func(t *testing.T) { + testJetStreamClusterWorkQueueStreamDiscardNewDesync(t, &nats.StreamConfig{ + Name: "WQTEST_MM", + Subjects: []string{"messages.*"}, + Replicas: 3, + MaxAge: 10 * time.Minute, + MaxMsgs: 100, + Retention: nats.WorkQueuePolicy, + Discard: nats.DiscardNew, + }) + }) + t.Run("max bytes", func(t *testing.T) { + testJetStreamClusterWorkQueueStreamDiscardNewDesync(t, &nats.StreamConfig{ + Name: "WQTEST_MB", + Subjects: []string{"messages.*"}, + Replicas: 3, + MaxAge: 10 * time.Minute, + MaxBytes: 1 * 1024 * 1024, + Retention: nats.WorkQueuePolicy, + Discard: nats.DiscardNew, + }) + }) +} + +func testJetStreamClusterWorkQueueStreamDiscardNewDesync(t *testing.T, sc *nats.StreamConfig) { + conf := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: { + store_dir: '%s', + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + system_account: sys + no_auth_user: js + accounts { + sys { + users = [ + { user: sys, pass: sys } + ] + } + js { + jetstream = enabled + users = [ + { user: js, pass: js } + ] + } + }` + c := createJetStreamClusterWithTemplate(t, conf, sc.Name, 3) + defer c.shutdown() + + nc, js := jsClientConnect(t, c.randomServer()) + defer nc.Close() + + cnc, cjs := jsClientConnect(t, c.randomServer()) + defer cnc.Close() + + _, err := js.AddStream(sc) + require_NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + psub, err := cjs.PullSubscribe("messages.*", "consumer") + require_NoError(t, err) + + stepDown := func() { + _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, sc.Name), nil, time.Second) + } + + // Messages will be produced and consumed in parallel, then once there are + // enough errors a leader election will be triggered. + var ( + wg sync.WaitGroup + received uint64 + errCh = make(chan error, 100_000) + receivedMap = make(map[string]*nats.Msg) + ) + wg.Add(1) + go func() { + tick := time.NewTicker(20 * time.Millisecond) + for { + select { + case <-ctx.Done(): + wg.Done() + return + case <-tick.C: + msgs, err := psub.Fetch(10, nats.MaxWait(200*time.Millisecond)) + if err != nil { + // The consumer will continue to timeout here eventually. + continue + } + for _, msg := range msgs { + received++ + receivedMap[msg.Subject] = msg + msg.Ack() + } + } + } + }() + + shouldDrop := make(map[string]error) + wg.Add(1) + go func() { + payload := []byte(strings.Repeat("A", 1024)) + tick := time.NewTicker(1 * time.Millisecond) + for i := 1; ; i++ { + select { + case <-ctx.Done(): + wg.Done() + return + case <-tick.C: + subject := fmt.Sprintf("messages.%d", i) + _, err := js.Publish(subject, payload, nats.RetryAttempts(0)) + if err != nil { + errCh <- err + } + // Capture the messages that have failed. + if err != nil { + shouldDrop[subject] = err + } + } + } + }() + + // Collect enough errors to cause things to get out of sync. + var errCount int +Setup: + for { + select { + case err = <-errCh: + errCount++ + if errCount%500 == 0 { + stepDown() + } else if errCount >= 2000 { + // Stop both producing and consuming. + cancel() + break Setup + } + case <-time.After(5 * time.Second): + // Unblock the test and continue. + cancel() + break Setup + } + } + + // Both goroutines should be exiting now.. + wg.Wait() + + // Let acks propagate for stream checks. + time.Sleep(250 * time.Millisecond) + + // Check messages that ought to have been dropped. + for subject := range receivedMap { + found, ok := shouldDrop[subject] + if ok { + t.Errorf("Should have dropped message published on %q since got error: %v", subject, found) + } + } +} diff --git a/server/jetstream_events.go b/server/jetstream_events.go index 7ea14ef858b..1852811bb96 100644 --- a/server/jetstream_events.go +++ b/server/jetstream_events.go @@ -193,10 +193,22 @@ const JSRestoreCompleteAdvisoryType = "io.nats.jetstream.advisory.v1.restore_com // Clustering specific. -// JSStreamLeaderElectedAdvisoryType is sent when the system elects a leader for a stream. +// JSClusterLeaderElectedAdvisoryType is sent when the system elects a new meta leader. +const JSDomainLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.domain_leader_elected" + +// JSClusterLeaderElectedAdvisory indicates that a domain has elected a new leader. +type JSDomainLeaderElectedAdvisory struct { + TypedEvent + Leader string `json:"leader"` + Replicas []*PeerInfo `json:"replicas"` + Cluster string `json:"cluster"` + Domain string `json:"domain,omitempty"` +} + +// JSStreamLeaderElectedAdvisoryType is sent when the system elects a new leader for a stream. const JSStreamLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.stream_leader_elected" -// JSStreamLeaderElectedAdvisory indicates that a stream has lost quorum and is stalled. +// JSStreamLeaderElectedAdvisory indicates that a stream has elected a new leader. type JSStreamLeaderElectedAdvisory struct { TypedEvent Account string `json:"account,omitempty"` @@ -222,7 +234,7 @@ type JSStreamQuorumLostAdvisory struct { // JSConsumerLeaderElectedAdvisoryType is sent when the system elects a leader for a consumer. const JSConsumerLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_leader_elected" -// JSConsumerLeaderElectedAdvisory indicates that a stream has lost quorum and is stalled. +// JSConsumerLeaderElectedAdvisory indicates that a consumer has elected a new leader. type JSConsumerLeaderElectedAdvisory struct { TypedEvent Account string `json:"account,omitempty"` diff --git a/server/jetstream_test.go b/server/jetstream_test.go index 5006d915161..9e4e4cd35bc 100644 --- a/server/jetstream_test.go +++ b/server/jetstream_test.go @@ -13534,6 +13534,13 @@ func TestJetStreamPurgeAndFilteredConsumers(t *testing.T) { t.Fatalf("Expected NumPending to be 10, got %d", ci.NumPending) } + // Also check unfiltered with interleaving messages. + _, err = js.AddConsumer("S", &nats.ConsumerConfig{ + Durable: "all", + AckPolicy: nats.AckExplicitPolicy, + }) + require_NoError(t, err) + // Now purge only adam. jr, _ := json.Marshal(&JSApiStreamPurgeRequest{Subject: "FOO.adam"}) _, err = nc.Request(fmt.Sprintf(JSApiStreamPurgeT, "S"), jr, time.Second) @@ -13559,6 +13566,12 @@ func TestJetStreamPurgeAndFilteredConsumers(t *testing.T) { if ci.AckFloor.Stream != 20 { t.Fatalf("Expected AckFloor for stream to be 20, got %d", ci.AckFloor.Stream) } + + ci, err = js.ConsumerInfo("S", "all") + require_NoError(t, err) + if ci.NumPending != 10 { + t.Fatalf("Expected NumPending to be 10, got %d", ci.NumPending) + } } // Issue #2662 @@ -22162,3 +22175,67 @@ func TestJetStreamConsumerNakThenAckFloorMove(t *testing.T) { require_Equal(t, ci.AckFloor.Stream, 11) require_Equal(t, ci.NumAckPending, 0) } + +func TestJetStreamSubjectFilteredPurgeClearsPendingAcks(t *testing.T) { + s := RunBasicJetStreamServer(t) + defer s.Shutdown() + + nc, js := jsClientConnect(t, s) + defer nc.Close() + + _, err := js.AddStream(&nats.StreamConfig{ + Name: "TEST", + Subjects: []string{"foo", "bar"}, + }) + require_NoError(t, err) + + for i := 0; i < 5; i++ { + js.Publish("foo", []byte("OK")) + js.Publish("bar", []byte("OK")) + } + + // Note that there are no subject filters here, this is deliberate + // as previously the purge with filter code was checking for them. + // We want to prove that unfiltered consumers also get purged. + ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ + Name: "my_consumer", + AckPolicy: nats.AckExplicitPolicy, + MaxAckPending: 10, + }) + require_NoError(t, err) + require_Equal(t, ci.NumPending, 10) + require_Equal(t, ci.NumAckPending, 0) + + sub, err := js.PullSubscribe(">", "", nats.Bind("TEST", "my_consumer")) + require_NoError(t, err) + + msgs, err := sub.Fetch(10) + require_NoError(t, err) + require_Len(t, len(msgs), 10) + + ci, err = js.ConsumerInfo("TEST", "my_consumer") + require_NoError(t, err) + require_Equal(t, ci.NumPending, 0) + require_Equal(t, ci.NumAckPending, 10) + + require_NoError(t, js.PurgeStream("TEST", &nats.StreamPurgeRequest{ + Subject: "foo", + })) + + ci, err = js.ConsumerInfo("TEST", "my_consumer") + require_NoError(t, err) + require_Equal(t, ci.NumPending, 0) + require_Equal(t, ci.NumAckPending, 5) + + for i := 0; i < 5; i++ { + js.Publish("foo", []byte("OK")) + } + msgs, err = sub.Fetch(5) + require_NoError(t, err) + require_Len(t, len(msgs), 5) + + ci, err = js.ConsumerInfo("TEST", "my_consumer") + require_NoError(t, err) + require_Equal(t, ci.NumPending, 0) + require_Equal(t, ci.NumAckPending, 10) +} diff --git a/server/jwt_test.go b/server/jwt_test.go index e5bbd223dcc..28314150f6f 100644 --- a/server/jwt_test.go +++ b/server/jwt_test.go @@ -5467,11 +5467,8 @@ func TestJWTJetStreamTiers(t *testing.T) { require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum consumers limit reached") _, err = js.Publish("testR1-3", msg[:]) - require_NoError(t, err) - _, err = js.Publish("testR1-3", []byte("1")) require_Error(t, err) require_Equal(t, err.Error(), "nats: resource limits exceeded for account") - } func TestJWTJetStreamMaxAckPending(t *testing.T) { @@ -5629,8 +5626,6 @@ func TestJWTJetStreamMaxStreamBytes(t *testing.T) { require_NoError(t, err) // test if we can push more messages into the stream - _, err = js.Publish("baz", msg[:]) - require_NoError(t, err) _, err = js.Publish("baz", msg[:]) // exceeds max stream bytes require_Error(t, err) require_Equal(t, err.Error(), "nats: resource limits exceeded for account") diff --git a/server/mqtt.go b/server/mqtt.go index 793d57d2736..a1abcb3ef7f 100644 --- a/server/mqtt.go +++ b/server/mqtt.go @@ -234,6 +234,8 @@ type mqttSessionManager struct { sessions map[string]*mqttAccountSessionManager // key is account name } +var testDisableRMSCache = false + type mqttAccountSessionManager struct { mu sync.RWMutex sessions map[string]*mqttSession // key is MQTT client ID @@ -243,7 +245,7 @@ type mqttAccountSessionManager struct { flapTimer *time.Timer // Timer to perform some cleanup of the flappers map sl *Sublist // sublist allowing to find retained messages for given subscription retmsgs map[string]*mqttRetainedMsgRef // retained messages - rmsCache sync.Map // map[subject]*mqttRetainedMsg + rmsCache *sync.Map // map[subject]mqttRetainedMsg jsa mqttJSA rrmLastSeq uint64 // Restore retained messages expected last sequence rrmDoneCh chan struct{} // To notify the caller that all retained messages have been loaded @@ -1142,6 +1144,9 @@ func (s *Server) mqttCreateAccountSessionManager(acc *Account, quitCh chan struc quitCh: quitCh, }, } + if !testDisableRMSCache { + as.rmsCache = &sync.Map{} + } // TODO record domain name in as here // The domain to communicate with may be required for JS calls. @@ -1236,10 +1241,12 @@ func (s *Server) mqttCreateAccountSessionManager(acc *Account, quitCh chan struc }) // Start the go routine that will clean up cached retained messages that expired. - s.startGoRoutine(func() { - defer s.grWG.Done() - as.cleanupRetainedMessageCache(s, closeCh) - }) + if as.rmsCache != nil { + s.startGoRoutine(func() { + defer s.grWG.Done() + as.cleanupRetainedMessageCache(s, closeCh) + }) + } lookupStream := func(stream, txt string) (*StreamInfo, error) { si, err := jsa.lookupStream(stream) @@ -1444,21 +1451,12 @@ func (s *Server) mqttCreateAccountSessionManager(acc *Account, quitCh chan struc rmLegacyDurName := mqttRetainedMsgsStreamName + "_" + jsa.id jsa.deleteConsumer(mqttRetainedMsgsStreamName, rmLegacyDurName) - // Using ephemeral consumer is too risky because if this server were to be - // disconnected from the rest for few seconds, then the leader would remove - // the consumer, so even after a reconnect, we would no longer receive - // retained messages. - // - // So we use a durable consumer, and create a new one each time we start. - // The old one should expire and get deleted due to inactivity. The name for - // the durable is $MQTT_rmsgs_{uuid}_{server-name}, the server name is just - // for readability. - rmDurName := mqttRetainedMsgsStreamName + "_" + nuid.Next() + "_" + s.String() - + // Create a new, uniquely names consumer for retained messages for this + // server. The prior one will expire eventually. ccfg := &CreateConsumerRequest{ Stream: mqttRetainedMsgsStreamName, Config: ConsumerConfig{ - Durable: rmDurName, + Name: mqttRetainedMsgsStreamName + "_" + nuid.Next(), FilterSubject: mqttRetainedMsgsStreamSubject + ">", DeliverSubject: rmsubj, ReplayPolicy: ReplayInstant, @@ -1466,7 +1464,7 @@ func (s *Server) mqttCreateAccountSessionManager(acc *Account, quitCh chan struc InactiveThreshold: 5 * time.Minute, }, } - if _, err := jsa.createConsumer(ccfg); err != nil { + if _, err := jsa.createEphemeralConsumer(ccfg); err != nil { return nil, fmt.Errorf("create retained messages consumer for account %q: %v", accName, err) } @@ -1646,17 +1644,26 @@ func (jsa *mqttJSA) sendAck(ackSubject string) { jsa.sendq.push(&mqttJSPubMsg{subj: ackSubject, hdr: -1}) } -func (jsa *mqttJSA) createConsumer(cfg *CreateConsumerRequest) (*JSApiConsumerCreateResponse, error) { +func (jsa *mqttJSA) createEphemeralConsumer(cfg *CreateConsumerRequest) (*JSApiConsumerCreateResponse, error) { cfgb, err := json.Marshal(cfg) if err != nil { return nil, err } - var subj string - if cfg.Config.Durable != _EMPTY_ { - subj = fmt.Sprintf(JSApiDurableCreateT, cfg.Stream, cfg.Config.Durable) - } else { - subj = fmt.Sprintf(JSApiConsumerCreateT, cfg.Stream) + subj := fmt.Sprintf(JSApiConsumerCreateT, cfg.Stream) + ccri, err := jsa.newRequest(mqttJSAConsumerCreate, subj, 0, cfgb) + if err != nil { + return nil, err + } + ccr := ccri.(*JSApiConsumerCreateResponse) + return ccr, ccr.ToError() +} + +func (jsa *mqttJSA) createDurableConsumer(cfg *CreateConsumerRequest) (*JSApiConsumerCreateResponse, error) { + cfgb, err := json.Marshal(cfg) + if err != nil { + return nil, err } + subj := fmt.Sprintf(JSApiDurableCreateT, cfg.Stream, cfg.Config.Durable) ccri, err := jsa.newRequest(mqttJSAConsumerCreate, subj, 0, cfgb) if err != nil { return nil, err @@ -2243,9 +2250,7 @@ func (as *mqttAccountSessionManager) handleRetainedMsg(key string, rf *mqttRetai // Update the in-memory retained message cache but only for messages // that are already in the cache, i.e. have been (recently) used. - if rm != nil { - as.setCachedRetainedMsg(key, rm, true, copyBytesToCache) - } + as.setCachedRetainedMsg(key, rm, true, copyBytesToCache) return } } @@ -2269,7 +2274,9 @@ func (as *mqttAccountSessionManager) handleRetainedMsgDel(subject string, seq ui as.sl = NewSublistWithCache() } if erm, ok := as.retmsgs[subject]; ok { - as.rmsCache.Delete(subject) + if as.rmsCache != nil { + as.rmsCache.Delete(subject) + } if erm.sub != nil { as.sl.Remove(erm.sub) erm.sub = nil @@ -2770,6 +2777,7 @@ func (as *mqttAccountSessionManager) loadRetainedMessages(subjects map[string]st w.Warnf("failed to decode retained message for subject %q: %v", ss[i], err) continue } + // Add the loaded retained message to the cache, and to the results map. key := ss[i][len(mqttRetainedMsgsStreamSubject):] as.setCachedRetainedMsg(key, &rm, false, false) @@ -2960,6 +2968,9 @@ func (as *mqttAccountSessionManager) transferRetainedToPerKeySubjectStream(log * } func (as *mqttAccountSessionManager) getCachedRetainedMsg(subject string) *mqttRetainedMsg { + if as.rmsCache == nil { + return nil + } v, ok := as.rmsCache.Load(subject) if !ok { return nil @@ -2973,6 +2984,9 @@ func (as *mqttAccountSessionManager) getCachedRetainedMsg(subject string) *mqttR } func (as *mqttAccountSessionManager) setCachedRetainedMsg(subject string, rm *mqttRetainedMsg, onlyReplace bool, copyBytesToCache bool) { + if as.rmsCache == nil || rm == nil { + return + } rm.expiresFromCache = time.Now().Add(mqttRetainedCacheTTL) if onlyReplace { if _, ok := as.rmsCache.Load(subject); !ok { @@ -4927,7 +4941,7 @@ func (sess *mqttSession) ensurePubRelConsumerSubscription(c *client) error { if opts.MQTT.ConsumerInactiveThreshold > 0 { ccr.Config.InactiveThreshold = opts.MQTT.ConsumerInactiveThreshold } - if _, err := sess.jsa.createConsumer(ccr); err != nil { + if _, err := sess.jsa.createDurableConsumer(ccr); err != nil { c.Errorf("Unable to add JetStream consumer for PUBREL for client %q: err=%v", id, err) return err } @@ -5033,7 +5047,7 @@ func (sess *mqttSession) processJSConsumer(c *client, subject, sid string, if opts.MQTT.ConsumerInactiveThreshold > 0 { ccr.Config.InactiveThreshold = opts.MQTT.ConsumerInactiveThreshold } - if _, err := sess.jsa.createConsumer(ccr); err != nil { + if _, err := sess.jsa.createDurableConsumer(ccr); err != nil { c.Errorf("Unable to add JetStream consumer for subscription on %q: err=%v", subject, err) return nil, nil, err } diff --git a/server/mqtt_ex_test.go b/server/mqtt_ex_test.go new file mode 100644 index 00000000000..44025bef806 --- /dev/null +++ b/server/mqtt_ex_test.go @@ -0,0 +1,424 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_mqtt_tests +// +build !skip_mqtt_tests + +package server + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "strconv" + "testing" + "time" +) + +func TestMQTTExCompliance(t *testing.T) { + mqttPath := os.Getenv("MQTT_CLI") + if mqttPath == "" { + if p, err := exec.LookPath("mqtt"); err == nil { + mqttPath = p + } + } + if mqttPath == "" { + t.Skip(`"mqtt" command is not found in $PATH nor $MQTT_CLI. See https://hivemq.github.io/mqtt-cli/docs/installation/#debian-package for installation instructions`) + } + + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + server_name: mqtt + jetstream { + store_dir = %q + } + mqtt { + listen: 127.0.0.1:-1 + } + `, t.TempDir()))) + s, o := RunServerWithConfig(conf) + defer testMQTTShutdownServer(s) + + cmd := exec.Command(mqttPath, "test", "-V", "3", "-p", strconv.Itoa(o.MQTT.Port)) + + output, err := cmd.CombinedOutput() + t.Log(string(output)) + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + t.Fatalf("mqtt cli exited with error: %v", exitError) + } + } +} + +const ( + KB = 1024 +) + +type mqttBenchMatrix struct { + QOS []int + MessageSize []int + Topics []int + Publishers []int + Subscribers []int +} + +type mqttBenchContext struct { + QOS int + MessageSize int + Topics int + Publishers int + Subscribers int + + Host string + Port int + + // full path to mqtt-test command + testCmdPath string +} + +var mqttBenchDefaultMatrix = mqttBenchMatrix{ + QOS: []int{0, 1, 2}, + MessageSize: []int{100, 1 * KB, 10 * KB}, + Topics: []int{100}, + Publishers: []int{1}, + Subscribers: []int{1}, +} + +type MQTTBenchmarkResult struct { + Ops int `json:"ops"` + NS map[string]time.Duration `json:"ns"` + Bytes int64 `json:"bytes"` +} + +func BenchmarkMQTTEx(b *testing.B) { + bc := mqttNewBenchEx(b) + b.Run("Server", func(b *testing.B) { + b.Cleanup(bc.startServer(b, false)) + bc.runAll(b) + }) + + b.Run("Cluster", func(b *testing.B) { + b.Cleanup(bc.startCluster(b, false)) + bc.runAll(b) + }) + + b.Run("Server-no-RMSCache", func(b *testing.B) { + b.Cleanup(bc.startServer(b, true)) + + bc.benchmarkSubRet(b) + }) + + b.Run("Cluster-no-RMSCache", func(b *testing.B) { + b.Cleanup(bc.startCluster(b, true)) + + bc.benchmarkSubRet(b) + }) +} + +func (bc mqttBenchContext) runAll(b *testing.B) { + bc.benchmarkPub(b) + bc.benchmarkPubRetained(b) + bc.benchmarkPubSub(b) + bc.benchmarkSubRet(b) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkPub(b *testing.B) { + m := mqttBenchDefaultMatrix. + NoSubscribers(). + NoTopics() + + b.Run("PUB", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runCommand(b, "pub", + "--qos", strconv.Itoa(bc.QOS), + "--n", strconv.Itoa(b.N), + "--size", strconv.Itoa(bc.MessageSize), + "--num-publishers", strconv.Itoa(bc.Publishers), + ) + }) + }) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkPubRetained(b *testing.B) { + // This bench is meaningless for QOS0 since the client considers the message + // sent as soon as it's written out. It is also useless for QOS2 since the + // flow takes a lot longer, and the difference of publishing as retained or + // not is lost in the noise. + m := mqttBenchDefaultMatrix. + NoSubscribers(). + NoTopics(). + QOS1Only() + + b.Run("PUBRET", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runCommand(b, "pub", "--retain", + "--qos", strconv.Itoa(bc.QOS), + "--n", strconv.Itoa(b.N), + "--size", strconv.Itoa(bc.MessageSize), + "--num-publishers", strconv.Itoa(bc.Publishers), + ) + }) + }) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkPubSub(b *testing.B) { + // This test uses a single built-in topic, and a built-in publisher, so no + // reason to run it for topics and publishers. + m := mqttBenchDefaultMatrix. + NoTopics(). + NoPublishers() + + b.Run("PUBSUB", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runCommand(b, "pubsub", + "--qos", strconv.Itoa(bc.QOS), + "--n", strconv.Itoa(b.N), + "--size", strconv.Itoa(bc.MessageSize), + "--num-subscribers", strconv.Itoa(bc.Subscribers), + ) + }) + }) +} + +// makes a copy of bc +func (bc mqttBenchContext) benchmarkSubRet(b *testing.B) { + // This test uses a a built-in publisher, and it makes most sense to measure + // the retained message delivery "overhead" on a QoS0 subscription; without + // the extra time involved in actually subscribing. + m := mqttBenchDefaultMatrix. + NoPublishers(). + QOS0Only() + + b.Run("SUBRET", func(b *testing.B) { + m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { + bc.runCommand(b, "subret", + "--qos", strconv.Itoa(bc.QOS), + "--n", strconv.Itoa(b.N), // number of subscribe requests + "--num-subscribers", strconv.Itoa(bc.Subscribers), + "--num-topics", strconv.Itoa(bc.Topics), + "--size", strconv.Itoa(bc.MessageSize), + ) + }) + }) +} + +func mqttBenchLookupCommand(b *testing.B, name string) string { + b.Helper() + cmd, err := exec.LookPath(name) + if err != nil { + b.Skipf("%q command is not found in $PATH. Please `go install github.com/nats-io/meta-nats/apps/go/mqtt/...@latest` and try again.", name) + } + return cmd +} + +func (bc mqttBenchContext) runCommand(b *testing.B, name string, extraArgs ...string) { + b.Helper() + + args := append([]string{ + name, + "-q", + "--servers", fmt.Sprintf("%s:%d", bc.Host, bc.Port), + }, extraArgs...) + + cmd := exec.Command(bc.testCmdPath, args...) + stdout, err := cmd.StdoutPipe() + if err != nil { + b.Fatalf("Error executing %q: %v", cmd.String(), err) + } + defer stdout.Close() + errbuf := bytes.Buffer{} + cmd.Stderr = &errbuf + if err = cmd.Start(); err != nil { + b.Fatalf("Error executing %q: %v", cmd.String(), err) + } + r := &MQTTBenchmarkResult{} + if err = json.NewDecoder(stdout).Decode(r); err != nil { + b.Fatalf("failed to decode output of %q: %v\n\n%s", cmd.String(), err, errbuf.String()) + } + if err = cmd.Wait(); err != nil { + b.Fatalf("Error executing %q: %v\n\n%s", cmd.String(), err, errbuf.String()) + } + + r.report(b) +} + +func (bc mqttBenchContext) initServer(b *testing.B) { + b.Helper() + bc.runCommand(b, "pubsub", + "--id", "__init__", + "--qos", "0", + "--n", "1", + "--size", "100", + "--num-subscribers", "1") +} + +func (bc *mqttBenchContext) startServer(b *testing.B, disableRMSCache bool) func() { + b.Helper() + b.StopTimer() + prevDisableRMSCache := testDisableRMSCache + testDisableRMSCache = disableRMSCache + o := testMQTTDefaultOptions() + s := testMQTTRunServer(b, o) + + o = s.getOpts() + bc.Host = o.MQTT.Host + bc.Port = o.MQTT.Port + bc.initServer(b) + return func() { + testMQTTShutdownServer(s) + testDisableRMSCache = prevDisableRMSCache + } +} + +func (bc *mqttBenchContext) startCluster(b *testing.B, disableRMSCache bool) func() { + b.Helper() + b.StopTimer() + prevDisableRMSCache := testDisableRMSCache + testDisableRMSCache = disableRMSCache + conf := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + mqtt { + listen: 127.0.0.1:-1 + stream_replicas: 3 + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } + ` + + cl := createJetStreamClusterWithTemplate(b, conf, "MQTT", 3) + o := cl.randomNonLeader().getOpts() + bc.Host = o.MQTT.Host + bc.Port = o.MQTT.Port + bc.initServer(b) + return func() { + cl.shutdown() + testDisableRMSCache = prevDisableRMSCache + } +} + +func mqttBenchWrapForMatrixField( + vFieldPtr *int, + arr []int, + f func(b *testing.B, bc *mqttBenchContext), + namef func(int) string, +) func(b *testing.B, bc *mqttBenchContext) { + if len(arr) == 0 { + return f + } + return func(b *testing.B, bc *mqttBenchContext) { + for _, value := range arr { + *vFieldPtr = value + b.Run(namef(value), func(b *testing.B) { + f(b, bc) + }) + } + } +} + +func (m mqttBenchMatrix) runMatrix(b *testing.B, bc mqttBenchContext, f func(*testing.B, *mqttBenchContext)) { + b.Helper() + f = mqttBenchWrapForMatrixField(&bc.MessageSize, m.MessageSize, f, func(size int) string { + return sizeKB(size) + }) + f = mqttBenchWrapForMatrixField(&bc.Topics, m.Topics, f, func(n int) string { + return fmt.Sprintf("%dtopics", n) + }) + f = mqttBenchWrapForMatrixField(&bc.Publishers, m.Publishers, f, func(n int) string { + return fmt.Sprintf("%dpubc", n) + }) + f = mqttBenchWrapForMatrixField(&bc.Subscribers, m.Subscribers, f, func(n int) string { + return fmt.Sprintf("%dsubc", n) + }) + f = mqttBenchWrapForMatrixField(&bc.QOS, m.QOS, f, func(qos int) string { + return fmt.Sprintf("QOS%d", qos) + }) + b.ResetTimer() + b.StartTimer() + f(b, &bc) +} + +func (m mqttBenchMatrix) NoSubscribers() mqttBenchMatrix { + m.Subscribers = nil + return m +} + +func (m mqttBenchMatrix) NoTopics() mqttBenchMatrix { + m.Topics = nil + return m +} + +func (m mqttBenchMatrix) NoPublishers() mqttBenchMatrix { + m.Publishers = nil + return m +} + +func (m mqttBenchMatrix) QOS0Only() mqttBenchMatrix { + m.QOS = []int{0} + return m +} + +func (m mqttBenchMatrix) QOS1Only() mqttBenchMatrix { + m.QOS = []int{1} + return m +} + +func sizeKB(size int) string { + unit := "B" + N := size + if size >= KB { + unit = "KB" + N = (N + KB/2) / KB + } + return fmt.Sprintf("%d%s", N, unit) +} + +func (r MQTTBenchmarkResult) report(b *testing.B) { + // Disable the default ns metric in favor of custom X_ms/op. + b.ReportMetric(0, "ns/op") + + // Disable MB/s since the github benchmarking action misinterprets the sign + // of the result (treats it as less is better). + b.SetBytes(0) + // b.SetBytes(r.Bytes) + + for unit, ns := range r.NS { + nsOp := float64(ns) / float64(r.Ops) + b.ReportMetric(nsOp/1000000, unit+"_ms/op") + } + + // Diable ReportAllocs() since it confuses the github benchmarking action + // with the noise. + // b.ReportAllocs() +} + +func mqttNewBenchEx(b *testing.B) *mqttBenchContext { + cmd := mqttBenchLookupCommand(b, "mqtt-test") + return &mqttBenchContext{ + testCmdPath: cmd, + } +} diff --git a/server/mqtt_test.go b/server/mqtt_test.go index 8fc4b874a35..efbfa2e49aa 100644 --- a/server/mqtt_test.go +++ b/server/mqtt_test.go @@ -7448,6 +7448,95 @@ func TestMQTTJetStreamRepublishAndQoS0Subscribers(t *testing.T) { testMQTTExpectNothing(t, r) } +// Test for auto-cleanup of consumers. +func TestMQTTDecodeRetainedMessage(t *testing.T) { + tdir := t.TempDir() + tmpl := ` + listen: 127.0.0.1:-1 + server_name: mqtt + jetstream { + store_dir = %q + } + + mqtt { + listen: 127.0.0.1:-1 + consumer_inactive_threshold: %q + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } + ` + conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, tdir, "0.2s"))) + s, o := RunServerWithConfig(conf) + defer testMQTTShutdownServer(s) + + // Connect and publish a retained message, this will be in the "newer" form, + // with the metadata in the header. + mc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) + defer mc.Close() + testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) + testMQTTPublish(t, mc, r, 0, false, true, "foo/1", 0, []byte("msg1")) + mc.Close() + + // Store a "legacy", JSON-encoded payload directly into the stream. + nc, js := jsClientConnect(t, s) + defer nc.Close() + + rm := mqttRetainedMsg{ + Origin: "test", + Subject: "foo.2", + Topic: "foo/2", + Flags: mqttPubFlagRetain, + Msg: []byte("msg2"), + } + jsonData, _ := json.Marshal(rm) + _, err := js.PublishMsg(&nats.Msg{ + Subject: mqttRetainedMsgsStreamSubject + rm.Subject, + Data: jsonData, + }) + if err != nil { + t.Fatalf("Error publishing retained message to JS directly: %v", err) + } + + // Restart the server to see that it picks up both retained messages on restart. + s.Shutdown() + s = RunServer(o) + defer testMQTTShutdownServer(s) + + // Connect again, subscribe, and check that we get both messages. + mc, r = testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) + defer mc.Close() + testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) + testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/+", qos: 0}}, []byte{0}) + for i := 0; i < 2; i++ { + b, pl := testMQTTReadPacket(t, r) + if pt := b & mqttPacketMask; pt != mqttPacketPub { + t.Fatalf("Expected PUBLISH packet %x, got %x", mqttPacketPub, pt) + } + _, _, topic := testMQTTGetPubMsgExEx(t, mc, r, mqttPubFlagRetain, pl, "", nil) + if string(topic) != "foo/1" && string(topic) != "foo/2" { + t.Fatalf("Expected foo/1 or foo/2, got %q", topic) + } + } + testMQTTExpectNothing(t, r) + mc.Close() + + // Clear both retained messages. + mc, r = testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) + defer mc.Close() + testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) + testMQTTPublish(t, mc, r, 0, false, true, "foo/1", 0, []byte{}) + testMQTTPublish(t, mc, r, 0, false, true, "foo/2", 0, []byte{}) + mc.Close() + + // Connect again, subscribe, and check that we get nothing. + mc, r = testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) + defer mc.Close() + testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) + testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/+", qos: 0}}, []byte{0}) + testMQTTExpectNothing(t, r) +} + ////////////////////////////////////////////////////////////////////////// // // Benchmarks diff --git a/server/norace_test.go b/server/norace_test.go index 83c4ea8e077..de85203fa7b 100644 --- a/server/norace_test.go +++ b/server/norace_test.go @@ -9514,3 +9514,88 @@ func TestNoRaceJetStreamClusterBadRestartsWithHealthzPolling(t *testing.T) { return nil }) } + +func TestNoRaceJetStreamKVReplaceWithServerRestart(t *testing.T) { + c := createJetStreamClusterExplicit(t, "R3S", 3) + defer c.shutdown() + + nc, _ := jsClientConnect(t, c.randomServer()) + defer nc.Close() + // Shorten wait time for disconnects. + js, err := nc.JetStream(nats.MaxWait(time.Second)) + require_NoError(t, err) + + kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ + Bucket: "TEST", + Replicas: 3, + }) + require_NoError(t, err) + + createData := func(n int) []byte { + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] + } + return b + } + + _, err = kv.Create("foo", createData(160)) + require_NoError(t, err) + + ch := make(chan struct{}) + wg := sync.WaitGroup{} + + // For counting errors that should not happen. + errCh := make(chan error, 1024) + + wg.Add(1) + go func() { + defer wg.Done() + + var lastData []byte + var revision uint64 + + for { + select { + case <-ch: + return + default: + k, err := kv.Get("foo") + if err == nats.ErrKeyNotFound { + errCh <- err + } else if k != nil { + if lastData != nil && k.Revision() == revision && !bytes.Equal(lastData, k.Value()) { + errCh <- fmt.Errorf("data loss [%s][rev:%d] expected:[%q] is:[%q]\n", "foo", revision, lastData, k.Value()) + } + newData := createData(160) + if revision, err = kv.Update("foo", newData, k.Revision()); err == nil { + lastData = newData + } + } + } + } + }() + + // Wait a short bit. + time.Sleep(2 * time.Second) + for _, s := range c.servers { + s.Shutdown() + // Need to leave servers down for awhile to trigger bug properly. + time.Sleep(5 * time.Second) + s = c.restartServer(s) + c.waitOnServerHealthz(s) + } + + // Shutdown the go routine above. + close(ch) + // Wait for it to finish. + wg.Wait() + + if len(errCh) != 0 { + for err := range errCh { + t.Logf("Received err %v during test", err) + } + t.Fatalf("Encountered errors") + } +} diff --git a/server/stream.go b/server/stream.go index b2bef9a8622..e4b2381bfda 100644 --- a/server/stream.go +++ b/server/stream.go @@ -283,6 +283,7 @@ type stream struct { clMu sync.Mutex clseq uint64 clfs uint64 + inflight map[uint64]uint64 leader string lqsent time.Time catchups map[string]uint64 @@ -1998,12 +1999,13 @@ func (mset *stream) purge(preq *JSApiStreamPurgeRequest) (purged uint64, err err // Purge consumers. // Check for filtered purge. if preq != nil && preq.Subject != _EMPTY_ { - ss := store.FilteredState(state.FirstSeq, preq.Subject) + ss := store.FilteredState(fseq, preq.Subject) fseq = ss.First } mset.clsMu.RLock() for _, o := range mset.cList { + start := fseq o.mu.RLock() // we update consumer sequences if: // no subject was specified, we can purge all consumers sequences @@ -2013,10 +2015,15 @@ func (mset *stream) purge(preq *JSApiStreamPurgeRequest) (purged uint64, err err // or consumer filter subject is subset of purged subject, // but not the other way around. o.isEqualOrSubsetMatch(preq.Subject) + // Check if a consumer has a wider subject space then what we purged + var isWider bool + if !doPurge && preq != nil && o.isFilteredMatch(preq.Subject) { + doPurge, isWider = true, true + start = state.FirstSeq + } o.mu.RUnlock() if doPurge { - o.purge(fseq, lseq) - + o.purge(start, lseq, isWider) } } mset.clsMu.RUnlock() @@ -4242,10 +4249,10 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte, // Process additional msg headers if still present. var msgId string var rollupSub, rollupAll bool + isClustered := mset.isClustered() if len(hdr) > 0 { outq := mset.outq - isClustered := mset.isClustered() // Certain checks have already been performed if in clustered mode, so only check if not. if !isClustered { @@ -4478,6 +4485,22 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte, } } + // If clustered this was already checked and we do not want to check here and possibly introduce skew. + if !isClustered { + if exceeded, err := jsa.wouldExceedLimits(stype, tierName, mset.cfg.Replicas, subject, hdr, msg); exceeded { + if err == nil { + err = NewJSAccountResourcesExceededError() + } + s.RateLimitWarnf("JetStream resource limits exceeded for account: %q", accName) + if canRespond { + resp.PubAck = &PubAck{Stream: name} + resp.Error = err + response, _ = json.Marshal(resp) + mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) + } + } + } + // Store actual msg. if lseq == 0 && ts == 0 { seq, ts, err = store.StoreMsg(subject, hdr, msg) @@ -4519,28 +4542,6 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte, return err } - if exceeded, apiErr := jsa.limitsExceeded(stype, tierName, mset.cfg.Replicas); exceeded { - s.RateLimitWarnf("JetStream resource limits exceeded for account: %q", accName) - if canRespond { - resp.PubAck = &PubAck{Stream: name} - if apiErr == nil { - resp.Error = NewJSAccountResourcesExceededError() - } else { - resp.Error = apiErr - } - response, _ = json.Marshal(resp) - mset.outq.sendMsg(reply, response) - } - // If we did not succeed put those values back. - var state StreamState - mset.store.FastState(&state) - mset.lseq = state.LastSeq - mset.lmsgId = olmsgId - mset.mu.Unlock() - store.RemoveMsg(seq) - return nil - } - // If we have a msgId make sure to save. if msgId != _EMPTY_ { mset.storeMsgIdLocked(&ddentry{msgId, seq, ts}) diff --git a/test/configs/certs/tlsauth/ca.pem b/test/configs/certs/tlsauth/ca.pem index 33553027008..e30e6999808 100644 --- a/test/configs/certs/tlsauth/ca.pem +++ b/test/configs/certs/tlsauth/ca.pem @@ -1,21 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDaDCCAlCgAwIBAgIUWyR/qbLooFMu+VcvmQhLAjokntQwDQYJKoZIhvcNAQEL -BQAwTDEkMCIGA1UEChMbU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuMRAwDgYD -VQQLEwdOQVRTLmlvMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTkwMjA0MTk1MDAw -WhcNMjQwMjAzMTk1MDAwWjBMMSQwIgYDVQQKExtTeW5hZGlhIENvbW11bmljYXRp -b25zIEluYy4xEDAOBgNVBAsTB05BVFMuaW8xEjAQBgNVBAMTCWxvY2FsaG9zdDCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN9ryA3PTdAPjC2VQkjy9JXJ -bOq2GpvGU+2/gC3TNRXOPJ5ZVy4svV8C9VA9t8gIbQHTYMzBFxyGz0+a/9+DEXot -crcVvsqaE5mewU9yjifDqUCGqOn9fo/zsYwD96KYtukEZ73D1Pyv+7EmkHNYqBKB -4/1gY/7AuuBcNp5bSpC4isGySZlL0wDjURyjfInrbDdMZi3QK2lPZP1okLZG5SCX -7pQM9riHwnzN94HINTzLTUdjxDBrm0Av9HCEeGT+iXwtXIhNaTkxjEy3a6b2saVl -wcaqcZbdGmJVgoncNlA3+277BPOAfbw4X5nGATaWPWxStkqeuhSaxahbCLNJGJcC -AwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0O -BBYEFG8G2+G/R8ovyXZoCjtIco9u9hrLMA0GCSqGSIb3DQEBCwUAA4IBAQBmuKij -sa+RKEoSVrdUWYwAhQJd17I1crhyLjzk3c5k4cXSIUM0XlGK81GZdPRV5EVym7FN -n8rhjAYizFykFbIcmiUrNa73jm2QTdMiL8WEzywNB0/X+XSJd+I1VeWOvYJMPTiY -KH/vcNYugVeWUzn6EF+iWnlpS9IHxcDvm6yjMJ242+KQWO7DGkHzbadB/BcryAdz -v6oBlHTJoPqgHUwaHfnTfqCQPTaTACUSFGNEnLuuXvLbbhZlpmLHRoqBiwpa0YQW -1EAICjLa6q5vSDSBrYJL2tIZz2vv/powIWMU1tdGFSALtpMucUH5Opi0Eaa+3cQB -fvl1Mck/CPY8e4/j +MIIDBTCCAe2gAwIBAgIUYm7HlYXiXY8u67v2vKjU5LUCHDUwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MjRaFw0zNDAxMzEy +MTQ4MjRaMBIxEDAOBgNVBAMMB05BVFMgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9GEoJ84UWZTsnovuz3StEtri3NdKlOqzNCGvurd3rZgop/jZB +RtJXQz9ZxdKx1ARpDV1m3CrQe63UbBRXJxA2XGfmBQ/BPo3IPEXJOsNEs9x5RsSL +RiJ2re7jbWKeQv/ucQPdmLJumAp+TGAzdOM/AnDaVTSLPoARp/Va8Frs7iFfPpuJ +tObvux4qnb/hxS2z39MWjyeM0dVOmjGwx9opxcE0hNI5ZutkoNxpmRayZqJSe85V +BSPGsuBwgncvA2GWTNIGFfN2oxQhSuI8yM7+l/0+BHFWfm2G7/09tWDvFnWTSpTQ +VISM3+6Wh91c6qSd0wsIb8q6jADAD6H8yhT/AgMBAAGjUzBRMB0GA1UdDgQWBBQk +vMZGyNfVHU3oTUASSfYiT4arNTAfBgNVHSMEGDAWgBQkvMZGyNfVHU3oTUASSfYi +T4arNTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBxN8yaacok +1lyg7eTEv7/Sn4VGkfbuWm9ZQ3DPLKJhbe7uVxNChRrR8nKshwlGv1Wa8i0c0lJ3 +O+Uw24gjfhl/8zhFyxh4ijllj6/FVBNqsTvnGQqtKJP4h8kScUIH21mQ6JQAvh4e +RY1sjPwZp+6vvogSrgQQ32jBaa8vfzcL2wECvnT1YqePVZYuRqEBjIvyG0ALlmE9 +DqZ8gH+W8E5IVulLVJxnYArCT1dW5AyM2fBETLB3PAWvSBkaCBl6QR+hLuyeR4vT +m6Qx9EKr8MgIpiH7psnx8C9eF5j5HiwHfhwAdWD9W2tRzTxSZP2LJ9E+qjaohdLf +6NYxXL8AHa17 -----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/client-key.pem b/test/configs/certs/tlsauth/client-key.pem index 002d091fc9c..1727e457db7 100644 --- a/test/configs/certs/tlsauth/client-key.pem +++ b/test/configs/certs/tlsauth/client-key.pem @@ -1,27 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEAr5QLVY0FeeEO+mPQDSGZSael1cO0cfABww7r5fNkxFRipJL0 -4uOKsP359dGo/Ay8MWS2mlAooX1jRA28YWghNjgZwv05bbmcQ2cGo9/7TTWKxusk -zHYgYLc9WNbgJnSGNceykQxIgnI4S/pF+fxpoJbxTOu71tviky5SAP93IU8JpfB4 -aIpSo0KfA0wDpauYD1orjrXY7JMvL/NRLLYvkTvJaMeuEAWuW3UacCV1wfT91oQW -FBaFupsatNT0UPJ+eMrfG0Gg6cM9bfU+ZNZROQOOR0gAM9XD4knuOYQsvdKWrEku -N8bBqCPOhCIYYCtUan607phtQoIBuwyxrEkSJQIDAQABAoIBAFn+IJ0V7gOdVmcC -d+XzHbWB518cs0VfBhgrcr/nM/PpaLH/3OLaTAER/GeBsgKWqHMMswd/JIQ5V4LP -I4otrDA1KwclcaUK6MwnZ2DhcdYOJnZ0meTuewP3h8scP8GWIiA4ng74Y8Xws2hF -/E34kU9NbprFjP7Ar25O5Js8VZxNJBWZXZqd4znTj8d4eqbZghM12RaUNQ1YSgB9 -dC9a4siubAWw3mZXX9T8te/uWbo7gY52vaMlO1nV+waZzkqdF7xizAYzYW+woAu5 -lMRnUylRFKtuAUZGsDATElVKeirWT/kkAjslRzLw0daOYfe+c321bE9fFwFi5b3e -DHSwpbUCgYEAyydwejT59DHvrQ1HRnKCL4+rvrCtFlHoJiSOKX8F38tVjCwFstHO -on+AxJMU8Om528aJ2RDPDzy1Cqn+cRAmgIsvAwa2JyBGqkN9nXq4Ji8eBL42ZNzK -wbnnFfm+qkBNXXon1gSnMN+8w9zX1dmIsoGmunZ/Ew0EB1bDsBEcI1cCgYEA3UBD -QkvFu6SlYq8ts5RcIn5bGgPkvudbRTb2IDYVMtLmPdDQP1X6s4NFdHErqvfwyIez -2WQLjLjUfU8fylEKxZL9cdHF/FAa5brnr3DAVT1Gsthv8WFFUkuZ52uAte4mGTL0 -FnGaz6mqoIrM/uNGKAgozMZPrDFobqsh5maepOMCgYAO3rMn7srA6grOEuO9r1IC -IzUB/zKcKKCichiJxwdqCxsW6H3+SccjM8v8F3v36lO1V4HthoJxbhMeVbUPF4yJ -6iYlxY79rCof+lKufTYPbXF4DWgz18lrhqz4edBP6+b9yZwy2SJXvHi3qWmO+J49 -2qmWimfgwBokY2BtecMifwKBgHiiMknycISIGBi/dQamDLpN9LQxjUY9dPk/J2GW -u2YzsY/gy7rM0V2RZIxBrFKSz3k27GvKbbWzjUAppSa1m07wfznQ68dPkerSRsLU -kjmnqGWZNygAJkDhsa+JYOtRRvqUWpvmI0e4tazFIVKUbssi78P/GK/FXLCCpIAw -Ua2LAoGABl1qtut/+NyOXZHJVl8tZIzeJOWkuMoKf0dOWmOsQl0Si2tGeB/ExI+O -mnhqozoV3nBGE8AyzMnUh/C6/tH5be3w/y3pTJ4rayffnYsYgf7mySAopjIbpW7w -iORnunwB7qnzx8yIS6rK0kpyvfp0P+bIOTwT1nEbw9wnBSjmCO4= ------END RSA PRIVATE KEY----- \ No newline at end of file +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjAaejJ5wqwsst +7T71LQCdsbRpctZU2ZbTIOalEvHNikkdB2gIqxlmswxITFVtNp2IvVhcd4KaNGdu +a4uUc7BifG6XwFtmw54SPTlbFtZbr1y4mjF6+Wm3tV0XnMtM18wvkRm7wbYG5fz8 +BfWl4tbCDF8lnuC3YHFwVJst+o1HoZ4YNutDFJqMXkHuxEPsI5PmS3XHHpttPQGr +vKnoYR0y0Cf4y3LCfQCQ+bFe2JS335WmJy2KVm+dpN+L2ASO8VvV/YTltSy8f4HQ +3Dd/VjZtri7ANP61Xrrx4OJS6IRkPo3uxaR306oKyF9a4irX1KY+Vi1gpYOX9foc +mVumCWCrAgMBAAECggEARk+F7CG/QlCQjEhb0ixtqheHPr7KhYHvhTUZV5XC2Aow +fEWAEdEfnUVY5GyMopWewOcPUJ86JeK5xI69/7QhHnIWz/0oT7zMF4jyDwDcSGLt +RzE3a5heid/Aflli9cvVZqUbaPnm1rXoeBrn+PxN7xigB92uh1qhw7ay0tPSkdUL +sIA2fkouwAYIE5TDhKk4lF8Xg6KlYJEesFDGTVsaGL8/RH7VEH/Z7EssiHF60m4l +Oj6Vg7gxMB1UWQFw2/LDMrDw0bLVdAoOC0M7NLXlFdoeKPREvFpIqw8BWMF3NxRV +OpUiLB/tv6A7nqevaa8KoWGUlCVY2DkCkgCmg+K8pQKBgQDVtb3nEJrOR496BaLn +pHKV+pEQjlVfcy36CoTa2PblbjkgSyA6wL9Itpkw4OF/Q8dop0qlbqZM9gUdjEhB +6DfVINNMGx4SkWDtBvPwouCRIWRvZUXdVuZMgDpUY+5n2nEkUHbmSUEtPbhHn5dp +9Tn1dsjNjweDTx2Q6nDuKUlSVwKBgQDDQ1fyyp6qI6opeFkfOsZ78Yk2lM6tRSkI +JqwdOwZoSptZfiaoAQtmwU3zxioFgIv4FjbPCosJyej0Egv7oyMucZHPGhP6kZDJ +LxZ0Jp0kDYYFCmUjOEpDf0mSmFEwmyPDb8052To3EuA4anEQd1KURG/0yKE4efwP +0m3ml9R3zQKBgAh1cxjMPXRgvLsVsgb9KVPqYQeIurRWeMFm3S9UWyFlpXkzwAjT +TD7yi0m1/Pbuldv8kyXNJWPycO1keg+xw1P6QqLGiAAwJOf82Hbz23OjILiQB53l +LKRmhuiENBGEQeowDSS8TYoe4UZkeLfG7w5aL0SDnsaBwSfVP7cNh0ttAoGAEJDO +DVMTUuvjq9EB/pxF6o37Th4hyqFrcb2WLIStbnul4lnJfcdY6EbODjhpqD3XohyA +WeBTG2l90fcV/StB+Na5wBA+Uau31Nmh1gjQnBZpoFPZcLt90WwjGcTCXpVK23HI +v3emcLWxQBgHr5Xv85Q6y1GaG+h9cfowSLfo1qECgYEAmqGyMa9OZ44+zni3nq2A +JM2t4B5QJdG36R/2m4J7odUnmsaRVxw6JxaoF0CctFxGs/zdumwuhYeTfq6Lj/ly +N1kJbvJJpguN1h4sOOLEk646mZzK2h5bMxsmPU2kNZFf1EUlB+Ro8i1TDvGY26Qs +qgjd6RlSZFBmSfVS47ihEQA= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/client.pem b/test/configs/certs/tlsauth/client.pem index 9b70e353cb1..8c873e0b02b 100644 --- a/test/configs/certs/tlsauth/client.pem +++ b/test/configs/certs/tlsauth/client.pem @@ -1,21 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDhjCCAm6gAwIBAgIUfLE3jBpwGplOUDcpbxIsvRInDGQwDQYJKoZIhvcNAQEL -BQAwTDEkMCIGA1UEChMbU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuMRAwDgYD -VQQLEwdOQVRTLmlvMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTkwMjA0MTk1MTAw -WhcNMjQwMjAzMTk1MTAwWjAoMRAwDgYDVQQLEwdOQVRTLmlvMRQwEgYDVQQDEwtl -eGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK+UC1WN -BXnhDvpj0A0hmUmnpdXDtHHwAcMO6+XzZMRUYqSS9OLjirD9+fXRqPwMvDFktppQ -KKF9Y0QNvGFoITY4GcL9OW25nENnBqPf+001isbrJMx2IGC3PVjW4CZ0hjXHspEM -SIJyOEv6Rfn8aaCW8Uzru9bb4pMuUgD/dyFPCaXweGiKUqNCnwNMA6WrmA9aK461 -2OyTLy/zUSy2L5E7yWjHrhAFrlt1GnAldcH0/daEFhQWhbqbGrTU9FDyfnjK3xtB -oOnDPW31PmTWUTkDjkdIADPVw+JJ7jmELL3SlqxJLjfGwagjzoQiGGArVGp+tO6Y -bUKCAbsMsaxJEiUCAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAww -CgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUa1+2h/uPRLo1o6Ee -qOTA1hKmUlcwHwYDVR0jBBgwFoAUbwbb4b9Hyi/JdmgKO0hyj272GsswCwYDVR0R -BAQwAoIAMA0GCSqGSIb3DQEBCwUAA4IBAQBr50msBYEhIMj6b8QoBAbvdBhE9TeF -76k4PZpGVF6EoGVdyTXvO0LBNT7BFysIVgy51Bv5kyz7z9B6iOwnnTDhZIj0kGMA -KPkgyMHHRlL9iOXC/fCuf9dfBuq7u2oykILCI8VY6yzTvHtIWg0/wk6/2e13WmtU -nkI9ySxJGzaZJaVjV7UWkzzR1anwJ6Q0IaRohIByWcE43uIzNi1sA1U8cZ70C43t -JNSXp9mBodV2jLCM3bU9jpSyz8thH3ghogioG8obYSS22a+Ei4SRsmHK1B2fu6Uh -z8UJCtjq5lbVlPgZQlmIHAJaq/cK8nSccv/2KBHx5hOR8rIqwsAL6p46 ------END CERTIFICATE----- \ No newline at end of file +MIIDPjCCAiagAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipYwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ5MzRaFw0zNDAxMzEy +MTQ5MzRaMCgxEDAOBgNVBAsMB05BVFMuaW8xFDASBgNVBAMMC2V4YW1wbGUuY29t +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAowGnoyecKsLLLe0+9S0A +nbG0aXLWVNmW0yDmpRLxzYpJHQdoCKsZZrMMSExVbTadiL1YXHeCmjRnbmuLlHOw +Ynxul8BbZsOeEj05WxbWW69cuJoxevlpt7VdF5zLTNfML5EZu8G2BuX8/AX1peLW +wgxfJZ7gt2BxcFSbLfqNR6GeGDbrQxSajF5B7sRD7COT5kt1xx6bbT0Bq7yp6GEd +MtAn+Mtywn0AkPmxXtiUt9+VpictilZvnaTfi9gEjvFb1f2E5bUsvH+B0Nw3f1Y2 +ba4uwDT+tV668eDiUuiEZD6N7sWkd9OqCshfWuIq19SmPlYtYKWDl/X6HJlbpglg +qwIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93 +d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFA5GJsoT3uf5GCyL5KSFoyEuRVltMB8G +A1UdIwQYMBaAFCS8xkbI19UdTehNQBJJ9iJPhqs1MA0GCSqGSIb3DQEBCwUAA4IB +AQAmBOQOp3CR5/P27RUonsgf5MCzblt6tBPJBMiZwB8sJyl9fXCTkzdqD0rJQb7q +5s7p00rUXdeh13oRjFcoFuopMDCk3kBYnKJHXRREJNLHfp6CPUMKlg0GJUZ6v04A +V7gVuhvmynHrmlbibMwbgZtZMnRU3x8JjawEUsEhoj3O2Qfen3sNfaOBlnwVUCBQ +ygSHQ0Pto1kQS+1Pc5DCwnOZ/qh5lORPdO1MNKqeu8HiiSJfuaCrQQM9zm72CHHY +F755qy8OvWjwK0H9rCFBYSrAnYk/pTvXIeBsgNRlURS/qv1rqIEvAbXhRnw7oyvl +P4bYY4pcpk32Ir2mFQFRQnSh +-----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/client2-key.pem b/test/configs/certs/tlsauth/client2-key.pem index 9e74cd2f733..6ae5fe08955 100644 --- a/test/configs/certs/tlsauth/client2-key.pem +++ b/test/configs/certs/tlsauth/client2-key.pem @@ -1,27 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEApRvhQ78kWi/gN1DWuTrk0vhS6Zva1U1TdHvtp0lvzgYqy7Z7 -mIEkX6tVZxIV7RXrOWu+CGWtkCb83upQudYdsFqbtOSbhsmeupAsKPePYSRJuYaf -BShNOEQ+UP+xEQJOv9LCHGFz8ukcF9XXJAlo0c5e6PDLmLPbMM7NmyZlqLGkXEgZ -cZEEG+BaxfR75gYoODnxkmvpx8F9nXKpDlIRNP9A2iTIWna0AX/r7fsivvBJCwj9 -j3j9g5gmEurT5oVeCaya6N60NVSbMug4UF70NF2EAtw8TKIat73ATdgf+EFY4wO2 -xdceM9OMiurrnV3FMOHMfU148rdFbHb99zIupwIDAQABAoIBAQCRxUSj8Gzi5yP5 -EnkRPorqLG3fbEfPTJ7i18thh7ebWNyN0IXchiAcCwOypUgQcuqjXpl/hm2vOIzH -Lm6pM/4wRj70fWVGolluc31Zif/fjw88KjvZbNSIWc/+6VBmKPhn6WaRcgTRsLep -35U7bsdJfP9Uf8vw/NIHjH4Afe0A+rJQ5aX17PLvlBH44WBOGvMq2qPPWfb5PzX6 -mne+a4M0FdTjfyL3qiJW2LF8OOVVDM9p9roQfZ15eKzxzQy4EhU7jtB0Bp9oGsu+ -nh0Pwvev0VQ1PPvZoxKP8FgwHSt/wekuht4bbQT0rQRNgKuJ+H1zwVe9mFDYslLh -bnfoFwQZAoGBANVIPdzEDx3op9n4RBQ7zBSwQUG2xRg42VrbjYdFkTB52a8m553I -4XrUb6QMCqFo1Va3BpZvEXfQIHXYSssakC1b1R/FVcRqlg27wKyFdW4B0UierBjt -iUjUfto4+CZzjpmEJJJcYduLigLqC2/hzFcUFs3paDHXJJfZfmlbhtd9AoGBAMYt -nhnZzgxRfzB05zaqw/hiR+X9lDMElOYKz8GUoygLlQYzsBnoQ+YD4ZRvETL1WhgN -M2Kc3BHjPdMFNZyvoRXgrqg018KDjzCqlHH1iWXVfUgnOlTi9kX3UdEhR4HcYi8Z -EnilkBseLgpC0byOXSiPVAUt34qy++D27OlwYZ/zAoGAU2Fjtev8EPBEtqUlUFe0 -SB5D1MH0Oaz35FpS8SBUS4RHgv8Nq5S9+bwVTSfb/BA03yq8a5FOXe3C0u9VBiQD -W4g8QKhwCFK3CPVutMOUDgat39sQYspyUkOot/1vnfCtPfz4IzP0mdTqhosjH4FB -1oUnCScHsfxu9OJ1VhEPHS0CgYB+lYLIFlRDkAbC59kMFRVp4TT1lfyEfeex7LP5 -fTyeBo/gz0Eruy0rjc0X572/o/IxLLVmxrTXBCRoVoqBE7m75LELJf2u9CORPVPm -WqSxlCUa4lui/vm5hRkQkMZBD4jzdntS7sXWXHeh/D5Fx1V/49USHdQMnvi+IFsB -XNQuuwKBgE48AdaUfWoqbkR+vfMbqeTyAXOg7VfmT1m8dc8mNPTeaEjUwb4M3EaQ -fT2B+tIASHfIaqLWPqjCrLxCBz4P4+4dkTLlWN+7BTVgK4Ecie/pEg8+q3CEVV2n -Bt5/z4xQCUmj24eiZogRIZmK9OsXEy4vQk4YH09tFSpFa+WwjoVX ------END RSA PRIVATE KEY----- \ No newline at end of file +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMsHsIYdxGt8ws +eoxGQQs4K+AfvznwuW+Qbnx+eGTS+OSaLqL9u81JdXxheeAVsVjRtz5qP3X9Vu75 +gSQU+AVhFp689VNR41vaOsKvX2dzVCaWCEwv66CekAop89vyB6fm14uSr9IusixH +u6vfjuze2FZE5xcw5mSYMysqKTbM4DMjXFJXFg5mw1YC9zkh3zCWUZdmidafwOB6 +MiJkVZsAQjAwc5oaD4I/ubCdmWr8COYbbwUl32k+t0eQfj9cvd4+R7uBYVNWu1BN +g0RqNZZlIM4EVonc1J+eN7oGo57jLrwf4jCOkcg2wd6X62CuuQHhwddQek4KGthM +JUIH7Q9TAgMBAAECggEAE80u2cy9xomZUuQ4FcPNFg4IjImvTT5jMJG/sWxsNIyn +cNL6KZm1blnTQorLxs11TjRv8U9aVrvGOpTnrK+htZa+nIEPImjgRehRVS3hkCKf +6Pu8gxZEX5KHqS9SI8Ph1k8bzYD80E+kQPxC0Em/WH+NOPUyJSTkrmSk1FtQVdle +KFk0+dRm10guI/rcInPUjpY6Cv9nSiAhanRPAtAANUvwYxVEMKzx5ns/JnQfAYAK +gYTopXAT61VqW1jn4opjD+OqparjbK362q/spggHGPhSamXq8y/N9JadSx+5591u +o0oZm/UxefwP1CkE4Q6Pv9n3yiZOwVaOiLLVzmr6wQKBgQD4kC953if336Ga+BdB +Hj83qUk5U2Avp6qWLptd6z/VQXDwlL2oQaaXP83T6XfpXTMwDabUYkpo1JZrFOJl +s9eZfxIDa0eH6evphqp6s/V6EhZkI9lL59ptZlK1hv1glmFDfn7WB1Yo7d8GgUx/ +S7CKoBK2WaX8YNsW9fIunadJeQKBgQDS0EDR46KiQBrjvNK4UxdfERl2af+6Sf+J +BRlfj+YA9/zKEL3jZGWVa5rgrD8BJrTsa86QSSACsxQx5J9vPogKgogmghd9RQVd +tHvIV0C2C6zCmZIvNuzn5+wMV7bWdoABPgbuZGTpc/LCwm//1EbwdNCTom8ssE19 +NTIBb7t4KwKBgQDXOqKheAhLzkz1D1WzgSlkXSWWieeD3D8OBBVsYcPIOP4+k80V +4KML3KexkzvNynIEbg3DYcjktQ/6cP8I6Y0K0MkcRMyPl7I7Z+w+i41Hwlm5JIGI +BJ9Sk4OSw+yqsgxOkT3qvjeRAUhZLaS7pSKdJraNR1s/Ce8sFpM6YjD0oQKBgAUc +gYXVRBs0/LHq4R0Q/q8SZhCl70pgAu8ajYvwnD4HxTxM/Z2m0IO38TBjXL+1ZYuZ +7Y84BquqFeJDzc3PsVK36X8thk5GPyQPfTTVUL9ZNx4cxRuZ9FKHIAUIl2lJxD7D +dz2Od5fldMxeFIMabYHlAy2hMZrex3IyuPyp7dyzAoGAKCqmXStl7QzV6ovrjyJ1 +oMP3BcQhTe1CvOEW7pGce/IGxqB0p7gICIH4UUDEVpitgvGLkOGScsd1rS86Fn+/ +RUlQQdgRIwzrlF1USV1kJ/SUI+7BbhiXUeRUmKuzPLgLSq8K3xEW0z21juSvq64W +gtiiGjt3D3I47v7ecjJ9dIA= +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/client2.pem b/test/configs/certs/tlsauth/client2.pem index 7ebc211094b..edc66519864 100644 --- a/test/configs/certs/tlsauth/client2.pem +++ b/test/configs/certs/tlsauth/client2.pem @@ -1,21 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDgzCCAmugAwIBAgIUXSH0jKq+6x2WG4RHqN8tATdptokwDQYJKoZIhvcNAQEL -BQAwTDEkMCIGA1UEChMbU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuMRAwDgYD -VQQLEwdOQVRTLmlvMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTkwMjA0MTk1ODAw -WhcNMjQwMjAzMTk1ODAwWjAlMQ0wCwYDVQQLEwRDTkNGMRQwEgYDVQQDEwtleGFt -cGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKUb4UO/JFov -4DdQ1rk65NL4Uumb2tVNU3R77adJb84GKsu2e5iBJF+rVWcSFe0V6zlrvghlrZAm -/N7qULnWHbBam7Tkm4bJnrqQLCj3j2EkSbmGnwUoTThEPlD/sRECTr/Swhxhc/Lp -HBfV1yQJaNHOXujwy5iz2zDOzZsmZaixpFxIGXGRBBvgWsX0e+YGKDg58ZJr6cfB -fZ1yqQ5SETT/QNokyFp2tAF/6+37Ir7wSQsI/Y94/YOYJhLq0+aFXgmsmujetDVU -mzLoOFBe9DRdhALcPEyiGre9wE3YH/hBWOMDtsXXHjPTjIrq651dxTDhzH1NePK3 -RWx2/fcyLqcCAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYI -KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUoS4dpE8Slaffykf+cVSc -g7IXvcYwHwYDVR0jBBgwFoAUbwbb4b9Hyi/JdmgKO0hyj272GsswCwYDVR0RBAQw -AoIAMA0GCSqGSIb3DQEBCwUAA4IBAQChjRkAiIuEXco4AkdoLO4wSN0i0b/toZ9b -U6X91UPCOQMYGLqe81DFYh3JE/+YjrwQYZz5Yb/vRVBC2HmTYkBXdP/74kRu4LCz -cdiVimz4GF2cBfFdxadNEJTQ8GW0fPtOIVwDZtJlNwi7ep58uR9Zld6Zo7FLRSzx -PtzBP6eEtwMJtVCk6PFluA7MY7k4c/TUW8bK0m9ybHIB8nqKuSWhZQBLdOhISyBz -/12xzX3An1NUpUaJnnD6ypEyfd8nZC0oAFC6+SAUMBWxcWYvhE5zcMaZQ3YtJUiC -0gR5d0Z1sjPYsq4KPow7IaTnzu3+0nLjZUHdU9RMfehJAxgBm3x0 ------END CERTIFICATE----- \ No newline at end of file +MIIDOzCCAiOgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipcwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTUwMzlaFw0zNDAxMzEy +MTUwMzlaMCUxDTALBgNVBAsMBENOQ0YxFDASBgNVBAMMC2V4YW1wbGUuY29tMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzLB7CGHcRrfMLHqMRkELOCvg +H7858LlvkG58fnhk0vjkmi6i/bvNSXV8YXngFbFY0bc+aj91/Vbu+YEkFPgFYRae +vPVTUeNb2jrCr19nc1QmlghML+ugnpAKKfPb8gen5teLkq/SLrIsR7ur347s3thW +ROcXMOZkmDMrKik2zOAzI1xSVxYOZsNWAvc5Id8wllGXZonWn8DgejIiZFWbAEIw +MHOaGg+CP7mwnZlq/AjmG28FJd9pPrdHkH4/XL3ePke7gWFTVrtQTYNEajWWZSDO +BFaJ3NSfnje6BqOe4y68H+IwjpHINsHel+tgrrkB4cHXUHpOChrYTCVCB+0PUwID +AQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cu +ZXhhbXBsZS5jb20wHQYDVR0OBBYEFEbKWjY1gwvxJusV+M5wUn+7MKR5MB8GA1Ud +IwQYMBaAFCS8xkbI19UdTehNQBJJ9iJPhqs1MA0GCSqGSIb3DQEBCwUAA4IBAQAo +873zVMG6tfoPRUZ/kEJcPEIaLmaolALyLEx3sZHe/B8hszuuMecBEfa02HwTSlzq +fKrkME95LGE9D+4hxyPEqPTqruESShUvBFQIoTQxePAhhUG9icF4gqUpYvRHXMiR +xIyfH3/KojDlBXRfDOaoXEXshiXfcYqbeh2qFdaoN24Vyh6lkNa2K3SUDAtKVFiF +jjBtNXuH/IJ3EWbs5AOOy98QtBMlT7kmummJVeaRR4QUfnzFrlj5nMSIopoxDm4N +QeoQSC+63fce6ZJLGQqEFQNR6howBcDQ/8fMR/oLsJ1Hr9VshIsu3kGTk4RUqHI9 +ipGt1UTvVf/uMUzA3yoC +-----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/server-key.pem b/test/configs/certs/tlsauth/server-key.pem index 38615ad58d9..a9077a68aee 100644 --- a/test/configs/certs/tlsauth/server-key.pem +++ b/test/configs/certs/tlsauth/server-key.pem @@ -1,27 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEAv+qCDnauYy08xfxu5S9Lm9oRlhXn0XUS1S73WaA4Sgv+QYYR -rDmsgjXTwiJ0VEDO+IUBXtqO1NOUxfpjifGX6rTdtab+Kxe9hztCacSnaDKUd887 -qmDqmCXwKGxEsz7loEliUMs/vlEsyTwpU6NjcIlKqlsOC8Jcyd4zffpPFmQuOnXz -r59lHiX5/kAQUoAPdJHjk6AEwa8lEgMjuOWiilkQxRIHPUPiVgs0l4AMqJcdW5BP -zbKSWpn4mSNPQIY/2hTWNdroe6qOAqWbtxtV8IkaQQGoEhavRRdgb92LGv9tK3h8 -qW2x+Q/Ac+zhRdXdUgDN2I2FhIh3aTepS29y+QIDAQABAoIBAQCxRXSM16ONiKOy -XdIxcNZuR6gm8mUHXRTgRlnEN/LGsv1QmP6KD1wBiqbnk9vQV2zWskTp0QhOHoI1 -vWtkZ/zjl92ThYURWQSAfYSDHltkLBRn9swuPQd1MtX7AMcUquyAekiOSK/ApEqy -NxgVYb7gnHCTmzgGNKpw7QazPxr+mkEOSKVNZvygNITPA9lvuCZ8Ky8ctIqOJOPB -wbIzHx3R18RPB++NMJ9T+mE+2vNhfJ4z6qVnOGNDle4Z3R0+JJog9G/57xlOd5HW -aTkM4HkA2sBsqfi3MA/DfhRAg+I8d5NFHEXc7pCX1NdaiDKVNPE0gd8mDd5BhJit -/lF09kQBAoGBANO06LLQepJnM5MNxdI2SYAxUZ/imlWN50ffuZcOmOgn7J9/3Gv2 -VUcJ01JeJwir8Vf+dl/d/F/fYQ6npss+XNpJ+7BB6OsNI6CMC87VH1YlT3QqG5qg -lAnMWK6YCxD5lu44P0dbR+R/NmW9yf1UPJGS3CKNoGWW9njb3isc5/iJAoGBAOgR -mvlCa4WD9WRRxeMW5RoWWZ+2a3OUm5Iy1uENsKCZ4gwWKoy96lMmfUh01sGgFewh -wgnX1hbkZe+YDqLQl2FpeBu1fzic1pCSVFBByaI2HZjufoGuo84ny9J++yoNABge -FuQSWXB7JT2v4SKK0mjV1LXzJawPHmJXzJayCCrxAoGBAJ13UCXAn4rJrDjS47MJ -of3xsQ7FU5oTJFX3eGl8+Aqlt4Cjb+X1oVRnYIFBerMegTK8GHwR9yewVNa7qHo/ -9nx+zvA49e/vI/LEd/vt1ZMTyVdUApgunC31injCqmiD3NlviNGgeYbhgCqI0fbV -cv+sRoSE5yro8IbQsx1KMNhRAoGBAMo23QUpXSuAKol5v6b7QjKTGxFSERsreMvR -xO9h0HCA5jmF7xmoOtCtjyldtewOJEwXtk6BZimYZ0J5CvfQLrhRALmUUwDvmP9s -ok80pA/We7/QwScbF90BTFdlElI39ccOIQAnBQxAIdk4skI5GNME0E6jSkY8/krP -GpSNGRThAoGAQsILz87TekVmyZuSGObkbnbFtvFKSfOSCuYuHIJjKyd4lUZi/CDY -iEtnXKk1UHT00zvliRGMrqA6++ePBZuMtN+8oRrGyfVEgQaHlNKlBMNvIrMgbZj0 -UGRJsOFDn2+kGsOvMdLMZKMzfqKk2mNefl10Ti0wbNXpG0LQC7JlhZQ= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAgrLiRyYcP54+ +AlElXoMjhOMg+VZjkVnC4CVXk+3jmKqAzI6bum9QHBnSdN/ULaPFU4Vgoqt1Puv8 +J9snNlSSE4CcgCCmWihFIwSpIaXW/GWCZVvCQDn3QxAZt48vTtEI27tMsRTCk/tV +kuShJ7OxrJyXcntBWbS+TeSOMVIF/v6lqWGjaDauPK0cpesa/qLElaHNlIV0+pM+ +5b3LIcEpuaECVwu5n5c9m/qdX7ZyPF2x8Z2I0zO5nLyFxb3WolPjok/qnZbhC/Gp +OX459F6yCVBmjkakaJocV1Ue7V0dFB7u3aYW0dwa+7Zb0613ZHMWKfAOzJKOiZC5 +xYXBxO4DAgMBAAECggEAA+ILzXOfVneHPHuO4acuRvoIG/yoT1OgHcWoPl16ywBX +BBUPYh9Yqm/lEQrDDwRCpuX54HEEVO+WRY/gpEHv/I6z8k3FTgWu66h6qhZGW/eo +U02yRzdFBz9AV/xzWpvUEvUq3Uk/GkBvRxKyiN7RPMODpXktXxKTo9Kg/XTYnjIb +2RWzKdoeptX6q/RRdPkBXxYV5XIaZLXPy+VaGgI7QcwApmCcin20c5smtLmGE7TV +4hh6wxPwGEiDkfMQGnqWd+VdzbnZjZF2/4ZVCNAbJJaFffTxnoC0uGjE3ImAztmT +TgmFbdfO/5qHTkJmfZaAbmo461k1O2Vd/p6LCN1mhQKBgQDokSJxOMuYurf1STAb +om6E5lGXAYR/2TsBxzfWPvUlZDzaSZ7IU+/zpv9GTATc0mvdIpnT2kZvRoz7nbbR +LFGjc2d1ITRBuDEX+qDkX1puFyhoYPyaz0AuC3AMqg28e3Z1I8nq3Op1TUKAMr9d +ZdOfRUcL65OM+YHPZ197AVwC9wKBgQDT6FgGceqBr3XJDikjU+kJfyrZrcE7sfrE +3RIV6uMj6Myw4WpUpuH7hwqQ0xAir8UKHG6SDScymXtrJCoG1xJmGGDyoLt80VsF +fXVhzL8rhIl48TVWptRUiIeZ2dXvianLkLPlLhVUcTThpjjAAbnNYhod1bHygX12 +w+Bd8lkeVQKBgBYLS7x3qbS8Xht96HV2HAu02R77IdgMey9b9sr0BMCak7oNKGPM +sP3jYmcDZaKYv2iikvolwm9hvJNNC7sf/E0F71SG5TEliGHBe+apsySkRUw/hTIX +WvoCU4ifxdWLzlqkHcuJTR/5RshoBwOPV1PNeUKD/eRq8gb6wW4jXtlZAoGBAK2o +C3MEqcQrUSA53ZaY7jGdKDWJQgC0oyfvbyHNAuVro0sU/3lt5WWmTg9PGDsExjm6 +ARbpdoTt6Ilt8o72c5p9Qf2zoNHyE2CVZruF+egkzi/xo99mCj1YQZ/gN4T80MwE +wpf+wvYXa9m7yWf4QhbA3VwzwodUfMf2T4lN0KCdAoGBAM04ecwBXCNIXzTgJDeK +zxfI95lphcNywdhUNw6KJPDg4ozPu72XrVToAtl7l6+2RPygNhSj117002AAgz30 +l0cfZoogXmIhZt+R39/2e9y31k+GDdEmNtPLsvbLPQ+Ttnab87pUABn4UC1XXGhF +Vt/jfLEw8ZDjB+aO4kFyZPIQ +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/server-no-ou-key.pem b/test/configs/certs/tlsauth/server-no-ou-key.pem index 39ce9f2c21f..ea0dd7873ce 100644 --- a/test/configs/certs/tlsauth/server-no-ou-key.pem +++ b/test/configs/certs/tlsauth/server-no-ou-key.pem @@ -1,27 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA330XGNotGWellHcjsr2O8rL51Htvk6wtJAep8wsPL7KtZYqe -0GL6No3WSw/JVTaIf27e45tBTLSPrVk0qGzkBAXxpy6ywdWmVL2NhoZ5E0jeABTb -Ud3Lj6zL+ggM+BQq2xJh85fytSNJ9nEcPBHzbwGB6baKpE84872qQ2jH8vXzdEHI -kWseJGItJ5JAcXGBQ0DqpUTUhI9Nww+EWjOYbiFZg2cy8ZMgOwReEgioQrwrN5i1 -PPeIpRKk23cQjeDa50J6uQjj6rcAO8PcQLjB4VCuSo4y4RSCLK4N5JZm01oOndLZ -Viyo14Rk87zNfQVKAlBm+wGfoYfbPNCu2oDUDwIDAQABAoIBAQCKW8n509CJ4tXq -pRuPo7Uk4dKzTjvUY4qKvMflNJqRQAADVh9eCXi4X3UkgUB2pc05f20z4cE6eKpe -elSUVN2Q9BEYHFwEjn1sBvHgL84zNzBhQohJFZPZffPF9kf5KZsihy3m/dH/fDpk -/L+rHL3lojxIcX9Bm945X9lR5EOtUJhkZ2xJI37VHNN1iWvNcNwls/Ud+lwsxndt -Pj8tjPZ1e0ypMSJllgKi8X4+JBIfcOL7v5OmNwOx5YMuUYDRWT35I3Tz/M12o0BB -nrrDBeTRXq/netqBWEa15id4g7xSGvZQKoNOpqDjtQZU851DRQniM1hqjTZR9qj+ -Pdj3/41RAoGBAOH+OBNogdvlADW4lUCcbF0cKktNwsv4hsRFyRUs3Yz91ESLhCc8 -KKRdqTO1SB3wdt8Q+Dxfsdorjf51CX1hjTk7lHL8v0GhU3zjOsanACd0pDMQK1lv -fQVKvZHZms5UZD0G27l6yS/Dxg7fkDUkTcR1n0t4cJqpjP0OKU+mkyMZAoGBAP0p -vkP7D4ICO2el023xK14SR1I4RbgEO9h3+IbdWsOjhioB/WjvWsWBJYa5ihFYCaGN -x3nEl1J8xCy6ADjbtvSYcgTWKjcsABY6adlwmoBE/UlwFrr0VA5+qJCyRK/zJMjr -Y5pICmC2rV6T+C+jpEOGNd06fN+lbNHfcAXKTv1nAoGAe2Cxto7Qjn9IDQwXl62O -T4rn4DK0zWyCDrdWn1PeJHITJ9TPMihau9lSXaNzmrzD+OYnz7Yiv8wVejzlEGlo -kz1evyQTOj5b+QuI9BkKMYAxgJssP2hpZbE3K2AUbt6N1u9el7VcDtKf11DgRtLq -Df51F9vKBfXYvfK0RQLYw0kCgYBioanEGINBNpdoWT3XXpdzzhFFYjEfcV7ThmIo -QQNEp2f049OT13T478jsBUtaWH9gFrm5ojMGax+PAWRmwos0HlSFt964ogbioh1t -HqbDBJ3dx7LDYb+B6izIOvvxxPv232ZtzFVmuqUu7N1LyiiMOjSwHUJba7rKxY+C -YgCGTwKBgFLUxkEDszCMPKQRb2G7rt/D5Hat7gsBvkYifa8wPA57v20johtKDlGF -lo1AolMCsebwS9IwwMO5taenJrWVNnw+YeHkr9E7sHoOlxRWjJImxs30cGJAEdQg -xMANEnpb8OQd4t47dpBgMh5KXqgQqg3kGPO55ZJlb1wfQ1HqWLii ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDepPh4tWYw66pP +GCEgpYzBTW36/s7cRnsuqhkiUolc5RB//JYaRz/sbIlY7tM8Z21nmjU0IXvlKPUt ++6Winv7VqDU7S1gOssnIPbVhXGCb4fC3JGjQ2qD/pO5WfoUh8yYph9tFg07+zMyQ +XgJiE8JglKCeD7kjna0/lkk63ddQP0npbH5943oMvHDj7UjJyBdE8nWX1LXWAmYJ +Wxym6MDfjWTVnyzwqm8fkl/BHp9fh98Nf9U6/gpq0ckRUHL4prhMw0ChhBVjkBQ8 +CrXrp82TQZbeFPm3uWTV3xf5bnbeO87dgiXP47yvqVFSQI2cFyo2JzEq6LVAeVPT +dKzENXyFAgMBAAECggEAHlLdvKMIPhV65rbknCuwFgvTtOHLjtjSojJspe4T42EX +dDcUwpN9s1e9BS3R+2Ii1n98S5Nb6oQ/kHm7v4BkOPll9qN2ZNoY/XraH16Tkeed +/3OoCvob/3WZOJKW017ojbOBO+B8e9us6OTE8lK6oKjdj2mYz68ED6sKYkggsT8M +ah4KkzrX8xNiQwMRs0VMfXb1M8DUd+nWcamWdUytf3ZPFWxymzYzjjxDuisK4faV +Fuqwk2FRtOuyMeygC7KuPwjBtqXuFuHLYtOlibs5ZFG9Nwd9lWQEaVCnOlgW35zz +CVkdszfCgimCL/K42vcmbdnI3GUO/g3szw9Vy95sAQKBgQDx/PHZks4FdxXSkuxZ +u3Qjh5d/sG9NMK0QUNgF/MnORQjvmVfQOSXObiMxoXBBNW25St00mO0W+ZzXee5z +3ANw6cPgSXla29RKC4yVuAXwOn/kPnuASQxdoGWEKuZJqUKbpK+FDgFbK9cZlSd/ +VNS9eW2j00CCqXK0FQ7FvfCoBQKBgQDriUonm0EwCoNrBlbllOynFYjFwg2yu/ti +H/1SAdmTcD+NZFRwhBmSKxni2rYyMoyqkHN93t8Sz8ELIK4sEsA/XUpEioeGIwMG +HQSRga2XYUVafuI/TQRHeMke703rtbIN7dkjEiWsM1gHoPcTRxZU9hRGq0p89GHq +zOf4tuAqgQKBgA3ceVYHLLnvalaXh+ZT8IEggTMVPirjwOYQW29sXXrtRWfEFt2c +iGfcszNilfWGQ/S7LxSWNe58+dj16QzF64SKP2gXjVYBBZYAN1tUITLzhuPiGFzu +0kCCsY3yjyJlCaW0t0Ed3kIErtuOSabnixAXZopdzXIulp1uH1yAVsqpAoGBAL+7 +ua62FoGp/ULRHUm0SlTVFcqN5iK6Ha/KBKeOM/Ruan2Jz6bsEfjHt0HQ8oG4XoO2 +JR2woHyqvCV3y/C6rt6l7YAQGLRbqel/E6nzG0FggFljcn8/DZ20uFvDR/X5qWDn +XlvLOPmNrjo/kQGTW5172BOS+obvVQoTFT6Ed8SBAoGBAI5802KpwzgN9IjPM1Zf +rBZ6PshZ14PFxeQ2tOhvFcr+N+ceSY19lex+UyQn7F214Kz2U3n35zgNdLhdHLK6 +WokXp21WDqeg8n1cUrQ43yWqVX8cjdNx3P84B5Mdnb0nhbM2smUSuhrO+fOzr37A +JegHloScSH3BcemRFfK31+ba +-----END PRIVATE KEY----- diff --git a/test/configs/certs/tlsauth/server-no-ou.pem b/test/configs/certs/tlsauth/server-no-ou.pem index 5aebb7d2f27..2ed37dca31e 100644 --- a/test/configs/certs/tlsauth/server-no-ou.pem +++ b/test/configs/certs/tlsauth/server-no-ou.pem @@ -1,21 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIDhTCCAm2gAwIBAgIUbXHf4iAemXfIpLSWpRMkEVsdjy8wDQYJKoZIhvcNAQEL -BQAwTDEkMCIGA1UEChMbU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuMRAwDgYD -VQQLEwdOQVRTLmlvMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTkwMjE4MjE0MjAw -WhcNMjQwMjE3MjE0MjAwWjAUMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDffRcY2i0ZZ6WUdyOyvY7ysvnUe2+TrC0k -B6nzCw8vsq1lip7QYvo2jdZLD8lVNoh/bt7jm0FMtI+tWTSobOQEBfGnLrLB1aZU -vY2GhnkTSN4AFNtR3cuPrMv6CAz4FCrbEmHzl/K1I0n2cRw8EfNvAYHptoqkTzjz -vapDaMfy9fN0QciRax4kYi0nkkBxcYFDQOqlRNSEj03DD4RaM5huIVmDZzLxkyA7 -BF4SCKhCvCs3mLU894ilEqTbdxCN4NrnQnq5COPqtwA7w9xAuMHhUK5KjjLhFIIs -rg3klmbTWg6d0tlWLKjXhGTzvM19BUoCUGb7AZ+hh9s80K7agNQPAgMBAAGjgZYw -gZMwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD -AjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBSubJojZMv2Ithfumfx12lAhWyaSjAf -BgNVHSMEGDAWgBRvBtvhv0fKL8l2aAo7SHKPbvYayzAUBgNVHREEDTALgglsb2Nh -bGhvc3QwDQYJKoZIhvcNAQELBQADggEBACGDpZ9oxuuCB8ujf7YXMPw7Ae1WH7DL -pBtPNL99bnPLQd/6iv12TWBFal5xSTCxhN/exAqDEk36zCKIEk2LvY/VJHP8Y2si -A79PqrlzSptOxeEuQRZnk+FiWYLwelkvL66TnpUW3QwdCCj/vodabDlaq6eSMEF+ -+kwxWZUixYkWASwuPSd7xjgNNBvIjeGvZIZvyjgTwyzPx/hSEMET68lRAoeE1S7S -IHqLPp/UwvvI9qMzBzkOz/XOmptB2fG3a5BLefEErcjVqfqYNAAA5V58+pyRrCSO -nX8UaSmy42aHryV0eelTLIxFSeSOHycpEhY2Hxq75JG8eSNfRd9rSt8= +MIIDKjCCAhKgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipUwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MzJaFw0zNDAxMzEy +MTQ4MzJaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAN6k+Hi1ZjDrqk8YISCljMFNbfr+ztxGey6qGSJSiVzlEH/8 +lhpHP+xsiVju0zxnbWeaNTQhe+Uo9S37paKe/tWoNTtLWA6yycg9tWFcYJvh8Lck +aNDaoP+k7lZ+hSHzJimH20WDTv7MzJBeAmITwmCUoJ4PuSOdrT+WSTrd11A/Sels +fn3jegy8cOPtSMnIF0TydZfUtdYCZglbHKbowN+NZNWfLPCqbx+SX8Een1+H3w1/ +1Tr+CmrRyRFQcvimuEzDQKGEFWOQFDwKteunzZNBlt4U+be5ZNXfF/ludt47zt2C +Jc/jvK+pUVJAjZwXKjYnMSrotUB5U9N0rMQ1fIUCAwEAAaN2MHQwMgYDVR0RBCsw +KYIJbG9jYWxob3N0ggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tMB0GA1Ud +DgQWBBS/WuyC1fvhLL8rnKIXWFdQFnGpozAfBgNVHSMEGDAWgBQkvMZGyNfVHU3o +TUASSfYiT4arNTANBgkqhkiG9w0BAQsFAAOCAQEAVgdcRJgo95MEDgEACekziOHn +n86DdgNin4FDkL7Y2sBDrpej4B0jKPm2H/M4qdB2Z17Ru2TdcXOyk/sMc395GHsN +BAdKuAcysvQ+USR3UXasJmC/CvoKGBOmFf9/Jor8U4Rs01bkXSd6pW8ytT3kyMak +3r5tNugzRxpJvVDgjHlUkfhBoLeeCr+k1cN1OvR4cFhY6vxqS6GBdopFGC3DlnTL +LPetNhQCd+r2mH1RT/56aLLRawy76GkBEZm/+mg+mYjxN3J1hWibouF4ccutvxtt +h2/4PJNsXv5yt4wibazFixJ843KPdfw6pafXbYZsvvgNfvLrpp8beCUwHEYq+w== -----END CERTIFICATE----- diff --git a/test/configs/certs/tlsauth/server.pem b/test/configs/certs/tlsauth/server.pem index 7bdd9a73b70..0e9f92ae971 100644 --- a/test/configs/certs/tlsauth/server.pem +++ b/test/configs/certs/tlsauth/server.pem @@ -1,22 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDoTCCAomgAwIBAgIUKrcs29uAsjrZ53kR88U2UAiz2FgwDQYJKoZIhvcNAQEL -BQAwTDEkMCIGA1UEChMbU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuMRAwDgYD -VQQLEwdOQVRTLmlvMRIwEAYDVQQDEwlsb2NhbGhvc3QwHhcNMTkwMjE4MTgxOTAw -WhcNMjQwMjE3MTgxOTAwWjAwMRowGAYDVQQLExFOQVRTLmlvIE9wZXJhdG9yczES -MBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC -AQEAv+qCDnauYy08xfxu5S9Lm9oRlhXn0XUS1S73WaA4Sgv+QYYRrDmsgjXTwiJ0 -VEDO+IUBXtqO1NOUxfpjifGX6rTdtab+Kxe9hztCacSnaDKUd887qmDqmCXwKGxE -sz7loEliUMs/vlEsyTwpU6NjcIlKqlsOC8Jcyd4zffpPFmQuOnXzr59lHiX5/kAQ -UoAPdJHjk6AEwa8lEgMjuOWiilkQxRIHPUPiVgs0l4AMqJcdW5BPzbKSWpn4mSNP -QIY/2hTWNdroe6qOAqWbtxtV8IkaQQGoEhavRRdgb92LGv9tK3h8qW2x+Q/Ac+zh -RdXdUgDN2I2FhIh3aTepS29y+QIDAQABo4GWMIGTMA4GA1UdDwEB/wQEAwIFoDAd -BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNV -HQ4EFgQUsi295rOWX4epJovs+SFWKL0M9NEwHwYDVR0jBBgwFoAUbwbb4b9Hyi/J -dmgKO0hyj272GsswFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUA -A4IBAQBt+JD/POQavswoTezh7YtXaw2F3DFhOz3yGvcKXBG/wRafWPuEZCMjY1cx -EgjOxA5kHPcsMNjRtQGAuV+1EymWUhJCyUq109D4eSNzR3IwXP3FhVtZ71XxCl+c -qUatFWvfGaU30TAHwB/QcXhLsliEMOaKRr3yTgM2BaTjsaBSA0/s/JwQiHvF0n1R -YYP4G7BON8IzhjsD38KjUrlAXT24VzCcDyFDk2c434jQwOZVxqHidB3lBlOmjX1X -eRJcJx/YkE7ej70R2f2WhQfZkgeF3iX8Xa5a10vjIqXcI9DkVb3GwAjx6+XDekJI -j4uv6u+kCJxABz3EyggU4BEPEPNL +MIIDRjCCAi6gAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipgwDQYJKoZIhvcNAQEL +BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMjA0NTdaFw0zNDAxMzEy +MjA0NTdaMDAxGjAYBgNVBAsMEU5BVFMuaW8gT3BlcmF0b3JzMRIwEAYDVQQDDAls +b2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAgrLiRyYc +P54+AlElXoMjhOMg+VZjkVnC4CVXk+3jmKqAzI6bum9QHBnSdN/ULaPFU4Vgoqt1 +Puv8J9snNlSSE4CcgCCmWihFIwSpIaXW/GWCZVvCQDn3QxAZt48vTtEI27tMsRTC +k/tVkuShJ7OxrJyXcntBWbS+TeSOMVIF/v6lqWGjaDauPK0cpesa/qLElaHNlIV0 ++pM+5b3LIcEpuaECVwu5n5c9m/qdX7ZyPF2x8Z2I0zO5nLyFxb3WolPjok/qnZbh +C/GpOX459F6yCVBmjkakaJocV1Ue7V0dFB7u3aYW0dwa+7Zb0613ZHMWKfAOzJKO +iZC5xYXBxO4DAgMBAAGjdjB0MDIGA1UdEQQrMCmCCWxvY2FsaG9zdIILZXhhbXBs +ZS5jb22CD3d3dy5leGFtcGxlLmNvbTAdBgNVHQ4EFgQUcpTmScu/KZHhs0KbdxtN +GXndXBEwHwYDVR0jBBgwFoAUJLzGRsjX1R1N6E1AEkn2Ik+GqzUwDQYJKoZIhvcN +AQELBQADggEBAKdqG82z3JBim/hiGf4LZT90ZHhw7ngPT9HUV4jYRIk0ngJ37ogK +KCYW0UBCkugdf0elxcggjAsJZGlz+hW2j8MynEqJ9UU7jPPp4AKJqZHy5x49Y1iL +kFlJE5a3LFJUaVG4JeYMqTL2zDtoj+hk7QPPoz88moDUbOHg3HccObHlISelVPON +K/kvnJ2NfXImYkh7MusRxVuB4LcRRi5rwT0pOdtSPBCeSH96BOeCHTriPHGecgc4 +71tgSaELXPM1YnaM2WmXoGU1MZ7Dx6c2q97FI+SWgKfm7B1GQGyAghgKxlRyhfNj +UvCrbaZDInrMWpMo3+upIBWpHzfmJVvUcYI= -----END CERTIFICATE-----