From 5b29f81f645538ce695c7bc5c04fe69a0d328eba Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Sat, 23 Nov 2024 16:55:41 -0600 Subject: [PATCH 1/3] Improve very large meta snapshots by moving to goccy json pkg and utilize s2.Encode vs EncodeBetter wich is faster and compresses better on our meta snapshots. Signed-off-by: Derek Collison --- go.mod | 1 + go.sum | 2 + server/accounts_test.go | 4 +- server/auth_callout_test.go | 4 +- server/auth_test.go | 2 +- server/certidp/certidp.go | 2 +- server/client.go | 2 +- server/client_test.go | 7 ++-- server/consumer.go | 2 +- server/events.go | 3 +- server/events_test.go | 2 +- server/filestore.go | 2 +- server/filestore_test.go | 4 +- server/gateway.go | 3 +- server/gateway_test.go | 5 ++- server/jetstream.go | 2 +- server/jetstream_api.go | 4 +- server/jetstream_cluster.go | 4 +- server/jetstream_cluster_1_test.go | 2 +- server/jetstream_cluster_2_test.go | 2 +- server/jetstream_cluster_3_test.go | 2 +- server/jetstream_cluster_4_test.go | 2 +- server/jetstream_consumer_test.go | 4 +- server/jetstream_events.go | 5 ++- server/jetstream_helpers_test.go | 2 +- server/jetstream_jwt_test.go | 7 ++-- server/jetstream_super_cluster_test.go | 2 +- server/jetstream_test.go | 2 +- server/jetstream_versioning_test.go | 3 +- server/jwt_test.go | 4 +- server/leafnode.go | 2 +- server/monitor.go | 2 +- server/monitor_test.go | 4 +- server/mqtt.go | 4 +- server/mqtt_ex_test_test.go | 2 +- server/mqtt_test.go | 4 +- server/msgtrace.go | 3 +- server/msgtrace_test.go | 2 +- server/nkey_test.go | 9 +++-- server/norace_test.go | 56 +++++++++++++++++++++++++- server/ocsp_responsecache.go | 4 +- server/opts_test.go | 4 +- server/reload_test.go | 4 +- server/route.go | 2 +- server/routes_test.go | 4 +- server/server.go | 15 ++++--- server/server_test.go | 5 ++- server/stream.go | 2 +- server/tpm/js_ek_tpm_windows.go | 2 +- server/util.go | 5 ++- server/websocket_test.go | 5 +-- test/auth_test.go | 4 +- test/gateway_test.go | 4 +- test/leafnode_test.go | 2 +- test/monitor_test.go | 4 +- test/new_routes_test.go | 4 +- test/norace_test.go | 2 +- test/ocsp_peer_test.go | 8 ++-- test/ports_test.go | 4 +- test/proto_test.go | 4 +- test/route_discovery_test.go | 4 +- test/routes_test.go | 4 +- test/service_latency_test.go | 4 +- test/test.go | 7 ++-- 64 files changed, 173 insertions(+), 114 deletions(-) diff --git a/go.mod b/go.mod index b5ae741d872..b68188021cf 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22 toolchain go1.22.8 require ( + github.com/goccy/go-json v0.10.3 github.com/google/go-tpm v0.9.0 github.com/klauspost/compress v1.17.11 github.com/minio/highwayhash v1.0.3 diff --git a/go.sum b/go.sum index c0104bcae45..ab0590d9c77 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= diff --git a/server/accounts_test.go b/server/accounts_test.go index eca0374a78a..7ef470e87f8 100644 --- a/server/accounts_test.go +++ b/server/accounts_test.go @@ -1,4 +1,4 @@ -// Copyright 2018-2023 The NATS Authors +// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,7 +15,6 @@ package server import ( "encoding/base64" - "encoding/json" "fmt" "net/http" "strconv" @@ -25,6 +24,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/auth_callout_test.go b/server/auth_callout_test.go index a578f9a3185..ac799cd51ee 100644 --- a/server/auth_callout_test.go +++ b/server/auth_callout_test.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 The NATS Authors +// Copyright 2022-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,7 +16,6 @@ package server import ( "bytes" "crypto/x509" - "encoding/json" "encoding/pem" "errors" "fmt" @@ -29,6 +28,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/auth_test.go b/server/auth_test.go index 05c3402f7d6..4ec9030dc81 100644 --- a/server/auth_test.go +++ b/server/auth_test.go @@ -15,7 +15,6 @@ package server import ( "context" - "encoding/json" "fmt" "net" "net/url" @@ -25,6 +24,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" ) diff --git a/server/certidp/certidp.go b/server/certidp/certidp.go index a26618577be..e92aebabf8e 100644 --- a/server/certidp/certidp.go +++ b/server/certidp/certidp.go @@ -17,12 +17,12 @@ import ( "crypto/sha256" "crypto/x509" "encoding/base64" - "encoding/json" "fmt" "net/url" "strings" "time" + "github.com/goccy/go-json" "golang.org/x/crypto/ocsp" ) diff --git a/server/client.go b/server/client.go index fa2a9395337..ce4f0265852 100644 --- a/server/client.go +++ b/server/client.go @@ -17,7 +17,6 @@ import ( "bytes" "crypto/tls" "crypto/x509" - "encoding/json" "errors" "fmt" "io" @@ -33,6 +32,7 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/internal/fastrand" diff --git a/server/client_test.go b/server/client_test.go index 45419ce5ae5..4380112b47d 100644 --- a/server/client_test.go +++ b/server/client_test.go @@ -1,4 +1,4 @@ -// Copyright 2012-2022 The NATS Authors +// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,7 +16,7 @@ package server import ( "bufio" "bytes" - "encoding/json" + "crypto/tls" "fmt" "io" "math" @@ -31,8 +31,7 @@ import ( "testing" "time" - "crypto/tls" - + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/consumer.go b/server/consumer.go index a8f6ab0f7da..f9cb4c2395a 100644 --- a/server/consumer.go +++ b/server/consumer.go @@ -16,7 +16,6 @@ package server import ( "bytes" "encoding/binary" - "encoding/json" "errors" "fmt" "math/rand" @@ -29,6 +28,7 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/server/avl" "github.com/nats-io/nuid" "golang.org/x/time/rate" diff --git a/server/events.go b/server/events.go index 29aebce79dc..ff5a8dbc355 100644 --- a/server/events.go +++ b/server/events.go @@ -18,7 +18,6 @@ import ( "compress/gzip" "crypto/sha256" "crypto/x509" - "encoding/json" "errors" "fmt" "math/rand" @@ -30,8 +29,8 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" - "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server/certidp" "github.com/nats-io/nats-server/v2/server/pse" diff --git a/server/events_test.go b/server/events_test.go index b3c7a1ef604..8fc06bdab5f 100644 --- a/server/events_test.go +++ b/server/events_test.go @@ -16,7 +16,6 @@ package server import ( "bytes" "crypto/sha256" - "encoding/json" "errors" "fmt" "math/rand" @@ -30,6 +29,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/filestore.go b/server/filestore.go index 22ccbcec936..135786352e3 100644 --- a/server/filestore.go +++ b/server/filestore.go @@ -22,7 +22,6 @@ import ( "crypto/sha256" "encoding/binary" "encoding/hex" - "encoding/json" "errors" "fmt" "hash" @@ -40,6 +39,7 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" "github.com/minio/highwayhash" "github.com/nats-io/nats-server/v2/server/avl" diff --git a/server/filestore_test.go b/server/filestore_test.go index 32dab26a2e0..bae397c6ede 100644 --- a/server/filestore_test.go +++ b/server/filestore_test.go @@ -24,7 +24,6 @@ import ( "crypto/sha256" "encoding/binary" "encoding/hex" - "encoding/json" "errors" "fmt" "io" @@ -40,6 +39,7 @@ import ( "time" "unicode/utf8" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" "github.com/nats-io/nuid" ) @@ -1484,8 +1484,6 @@ func TestFileStoreMeta(t *testing.T) { if err := json.Unmarshal(buf, &oconfig2); err != nil { t.Fatalf("Error unmarshalling: %v", err) } - // Since we set name we will get that back now. - oconfig.Name = oname if !reflect.DeepEqual(oconfig2, oconfig) { t.Fatalf("Consumer configs not equal, got %+v vs %+v", oconfig2, oconfig) } diff --git a/server/gateway.go b/server/gateway.go index 1fa89c71d4b..e43e956bb97 100644 --- a/server/gateway.go +++ b/server/gateway.go @@ -18,7 +18,6 @@ import ( "cmp" "crypto/sha256" "crypto/tls" - "encoding/json" "errors" "fmt" "math/rand" @@ -30,6 +29,8 @@ import ( "sync" "sync/atomic" "time" + + "github.com/goccy/go-json" ) const ( diff --git a/server/gateway_test.go b/server/gateway_test.go index b397bfedc10..665c9684520 100644 --- a/server/gateway_test.go +++ b/server/gateway_test.go @@ -18,7 +18,6 @@ import ( "bytes" "context" "crypto/tls" - "encoding/json" "fmt" "net" "net/url" @@ -31,10 +30,12 @@ import ( "testing" "time" - . "github.com/nats-io/nats-server/v2/internal/ocsp" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/logger" "github.com/nats-io/nats.go" "golang.org/x/crypto/ocsp" + + . "github.com/nats-io/nats-server/v2/internal/ocsp" ) func init() { diff --git a/server/jetstream.go b/server/jetstream.go index 32305121a0d..13e81028db2 100644 --- a/server/jetstream.go +++ b/server/jetstream.go @@ -18,7 +18,6 @@ import ( "crypto/sha256" "encoding/binary" "encoding/hex" - "encoding/json" "fmt" "math" "os" @@ -30,6 +29,7 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/minio/highwayhash" "github.com/nats-io/nats-server/v2/server/sysmem" "github.com/nats-io/nats-server/v2/server/tpm" diff --git a/server/jetstream_api.go b/server/jetstream_api.go index f73d2f9c29b..36ef47660ae 100644 --- a/server/jetstream_api.go +++ b/server/jetstream_api.go @@ -1,4 +1,4 @@ -// Copyright 2020-2023 The NATS Authors +// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,7 +16,6 @@ package server import ( "bytes" "cmp" - "encoding/json" "errors" "fmt" "io" @@ -31,6 +30,7 @@ import ( "time" "unicode" + "github.com/goccy/go-json" "github.com/nats-io/nuid" ) diff --git a/server/jetstream_cluster.go b/server/jetstream_cluster.go index 0ee91b31e39..bdd364cf5ce 100644 --- a/server/jetstream_cluster.go +++ b/server/jetstream_cluster.go @@ -18,7 +18,6 @@ import ( "cmp" crand "crypto/rand" "encoding/binary" - "encoding/json" "errors" "fmt" "math" @@ -32,6 +31,7 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" "github.com/minio/highwayhash" "github.com/nats-io/nuid" @@ -1572,7 +1572,7 @@ func (js *jetStream) metaSnapshot() []byte { b, _ := json.Marshal(streams) js.mu.RUnlock() - return s2.EncodeBetter(nil, b) + return s2.Encode(nil, b) } func (js *jetStream) applyMetaSnapshot(buf []byte, ru *recoveryUpdates, isRecovering bool) error { diff --git a/server/jetstream_cluster_1_test.go b/server/jetstream_cluster_1_test.go index a4eb85d37e0..84502d1d2e6 100644 --- a/server/jetstream_cluster_1_test.go +++ b/server/jetstream_cluster_1_test.go @@ -20,7 +20,6 @@ import ( "bytes" "context" crand "crypto/rand" - "encoding/json" "errors" "fmt" "math/rand" @@ -33,6 +32,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" ) diff --git a/server/jetstream_cluster_2_test.go b/server/jetstream_cluster_2_test.go index bbfe5288df5..c4df4e68bda 100644 --- a/server/jetstream_cluster_2_test.go +++ b/server/jetstream_cluster_2_test.go @@ -22,7 +22,6 @@ import ( crand "crypto/rand" "encoding/binary" "encoding/hex" - "encoding/json" "errors" "fmt" "math/rand" @@ -37,6 +36,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats.go" ) diff --git a/server/jetstream_cluster_3_test.go b/server/jetstream_cluster_3_test.go index db370e53939..c3b858dc37f 100644 --- a/server/jetstream_cluster_3_test.go +++ b/server/jetstream_cluster_3_test.go @@ -19,7 +19,6 @@ package server import ( "bytes" "context" - "encoding/json" "errors" "fmt" "math/rand" @@ -33,6 +32,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" ) diff --git a/server/jetstream_cluster_4_test.go b/server/jetstream_cluster_4_test.go index 033c557d2bf..18478a28120 100644 --- a/server/jetstream_cluster_4_test.go +++ b/server/jetstream_cluster_4_test.go @@ -19,7 +19,6 @@ package server import ( "context" "encoding/binary" - "encoding/json" "errors" "fmt" "math/rand" @@ -35,6 +34,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats.go" "github.com/nats-io/nuid" ) diff --git a/server/jetstream_consumer_test.go b/server/jetstream_consumer_test.go index 2fd1aa45572..9b8a38fff1d 100644 --- a/server/jetstream_consumer_test.go +++ b/server/jetstream_consumer_test.go @@ -1,4 +1,4 @@ -// Copyright 2022-2023 The NATS Authors +// Copyright 2022-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,7 +17,6 @@ package server import ( - "encoding/json" "errors" "fmt" "math/rand" @@ -29,6 +28,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats.go" "github.com/nats-io/nuid" ) diff --git a/server/jetstream_events.go b/server/jetstream_events.go index 825fd5c378d..2afe48bfe9b 100644 --- a/server/jetstream_events.go +++ b/server/jetstream_events.go @@ -1,4 +1,4 @@ -// Copyright 2020-2021 The NATS Authors +// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,8 +14,9 @@ package server import ( - "encoding/json" "time" + + "github.com/goccy/go-json" ) func (s *Server) publishAdvisory(acc *Account, subject string, adv any) { diff --git a/server/jetstream_helpers_test.go b/server/jetstream_helpers_test.go index 1512012a0bf..7a60ce942ea 100644 --- a/server/jetstream_helpers_test.go +++ b/server/jetstream_helpers_test.go @@ -18,7 +18,6 @@ package server import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -33,6 +32,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats.go" "golang.org/x/time/rate" ) diff --git a/server/jetstream_jwt_test.go b/server/jetstream_jwt_test.go index 279cc984961..737b4fbb304 100644 --- a/server/jetstream_jwt_test.go +++ b/server/jetstream_jwt_test.go @@ -1,4 +1,4 @@ -// Copyright 2020-2023 The NATS Authors +// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,7 +17,6 @@ package server import ( - "encoding/json" "errors" "fmt" "net/http" @@ -27,9 +26,11 @@ import ( "testing" "time" - jwt "github.com/nats-io/jwt/v2" + "github.com/goccy/go-json" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" + + jwt "github.com/nats-io/jwt/v2" ) func TestJetStreamJWTLimits(t *testing.T) { diff --git a/server/jetstream_super_cluster_test.go b/server/jetstream_super_cluster_test.go index c2ec399041a..4e20d5aca28 100644 --- a/server/jetstream_super_cluster_test.go +++ b/server/jetstream_super_cluster_test.go @@ -17,7 +17,6 @@ package server import ( - "encoding/json" "errors" "fmt" "math/rand" @@ -30,6 +29,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/jetstream_test.go b/server/jetstream_test.go index 42c4f00f38e..0c14b78c983 100644 --- a/server/jetstream_test.go +++ b/server/jetstream_test.go @@ -21,7 +21,6 @@ import ( "context" crand "crypto/rand" "encoding/base64" - "encoding/json" "errors" "fmt" "io" @@ -42,6 +41,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server/sysmem" "github.com/nats-io/nats.go" diff --git a/server/jetstream_versioning_test.go b/server/jetstream_versioning_test.go index 360deda7465..6e1fb7b5fea 100644 --- a/server/jetstream_versioning_test.go +++ b/server/jetstream_versioning_test.go @@ -19,15 +19,14 @@ package server import ( "archive/tar" "bytes" - "encoding/json" "fmt" "reflect" "strconv" "testing" "time" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" - "github.com/nats-io/nats.go" ) diff --git a/server/jwt_test.go b/server/jwt_test.go index 8158a26e0b7..b25be97ad2d 100644 --- a/server/jwt_test.go +++ b/server/jwt_test.go @@ -1,4 +1,4 @@ -// Copyright 2018-2020 The NATS Authors +// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,7 +17,6 @@ import ( "bufio" "context" "encoding/base64" - "encoding/json" "errors" "fmt" "io" @@ -31,6 +30,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/leafnode.go b/server/leafnode.go index 021c8a9f1a0..b16ba598423 100644 --- a/server/leafnode.go +++ b/server/leafnode.go @@ -18,7 +18,6 @@ import ( "bytes" "crypto/tls" "encoding/base64" - "encoding/json" "fmt" "io" "math/rand" @@ -36,6 +35,7 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" "github.com/nats-io/nkeys" diff --git a/server/monitor.go b/server/monitor.go index 6d7b58721f9..ca65a670d35 100644 --- a/server/monitor.go +++ b/server/monitor.go @@ -20,7 +20,6 @@ import ( "crypto/tls" "crypto/x509" "encoding/hex" - "encoding/json" "expvar" "fmt" "net" @@ -37,6 +36,7 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server/pse" ) diff --git a/server/monitor_test.go b/server/monitor_test.go index 694f5ae4199..e04567f80c1 100644 --- a/server/monitor_test.go +++ b/server/monitor_test.go @@ -1,4 +1,4 @@ -// Copyright 2013-2023 The NATS Authors +// Copyright 2013-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,7 +16,6 @@ package server import ( "bytes" "crypto/tls" - "encoding/json" "errors" "fmt" "io" @@ -35,6 +34,7 @@ import ( "time" "unicode" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/mqtt.go b/server/mqtt.go index 1c2aa1405bc..69eff3c766b 100644 --- a/server/mqtt.go +++ b/server/mqtt.go @@ -1,4 +1,4 @@ -// Copyright 2020-2023 The NATS Authors +// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,7 +18,6 @@ import ( "cmp" "crypto/tls" "encoding/binary" - "encoding/json" "errors" "fmt" "io" @@ -31,6 +30,7 @@ import ( "time" "unicode/utf8" + "github.com/goccy/go-json" "github.com/nats-io/nuid" ) diff --git a/server/mqtt_ex_test_test.go b/server/mqtt_ex_test_test.go index 9acad558779..4280b714f99 100644 --- a/server/mqtt_ex_test_test.go +++ b/server/mqtt_ex_test_test.go @@ -18,7 +18,6 @@ package server import ( "bytes" - "encoding/json" "fmt" "io" "os" @@ -27,6 +26,7 @@ import ( "strings" "testing" + "github.com/goccy/go-json" "github.com/nats-io/nuid" ) diff --git a/server/mqtt_test.go b/server/mqtt_test.go index 7a72136570f..920e7c35976 100644 --- a/server/mqtt_test.go +++ b/server/mqtt_test.go @@ -1,4 +1,4 @@ -// Copyright 2020-2023 The NATS Authors +// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -20,7 +20,6 @@ import ( "bufio" "bytes" "crypto/tls" - "encoding/json" "errors" "fmt" "io" @@ -33,6 +32,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/msgtrace.go b/server/msgtrace.go index b7e229bab8f..2ac9a2f0d54 100644 --- a/server/msgtrace.go +++ b/server/msgtrace.go @@ -15,13 +15,14 @@ package server import ( "bytes" - "encoding/json" "fmt" "math/rand" "strconv" "strings" "sync/atomic" "time" + + "github.com/goccy/go-json" ) const ( diff --git a/server/msgtrace_test.go b/server/msgtrace_test.go index 5cefc9f4eb6..8b2117e3de4 100644 --- a/server/msgtrace_test.go +++ b/server/msgtrace_test.go @@ -19,7 +19,6 @@ package server import ( "bytes" "compress/gzip" - "encoding/json" "fmt" "io" "net" @@ -28,6 +27,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" diff --git a/server/nkey_test.go b/server/nkey_test.go index f593a2f7548..174f7067100 100644 --- a/server/nkey_test.go +++ b/server/nkey_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The NATS Authors +// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,15 +15,16 @@ package server import ( "bufio" - crand "crypto/rand" "encoding/base64" - "encoding/json" "fmt" - mrand "math/rand" "strings" "testing" "time" + crand "crypto/rand" + mrand "math/rand" + + "github.com/goccy/go-json" "github.com/nats-io/nkeys" ) diff --git a/server/norace_test.go b/server/norace_test.go index 8cb4e1f7e9c..e8f449a780f 100644 --- a/server/norace_test.go +++ b/server/norace_test.go @@ -22,7 +22,6 @@ import ( "compress/gzip" "context" "encoding/binary" - "encoding/json" "errors" "fmt" "io" @@ -48,6 +47,7 @@ import ( crand "crypto/rand" "crypto/sha256" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server/avl" @@ -11182,3 +11182,57 @@ func TestNoRaceJetStreamClusterCheckInterestStatePerformanceInterest(t *testing. mset.checkInterestState() require_True(t, time.Since(start) < elapsed/100) } + +func TestNoRaceJetStreamClusterLargeMetaSnapshotTiming(t *testing.T) { + // This test was to show improvements in speed for marshaling the meta layer with lots of assets. + // Move to S2.Encode vs EncodeBetter which is 2x faster and actually better compression. + // Also moved to goccy json which is faster then the default and in my tests now always matches + // the default encoder byte for byte which last time I checked it did not. + t.Skip() + + c := createJetStreamClusterExplicit(t, "R3F", 3) + defer c.shutdown() + + // Create 200 streams, each with 500 consumers. + numStreams := 200 + numConsumers := 500 + wg := sync.WaitGroup{} + wg.Add(numStreams) + for i := 0; i < numStreams; i++ { + go func() { + defer wg.Done() + s := c.randomServer() + nc, js := jsClientConnect(t, s) + defer nc.Close() + sname := fmt.Sprintf("TEST-SNAPSHOT-%d", i) + subj := fmt.Sprintf("foo.%d", i) + _, err := js.AddStream(&nats.StreamConfig{ + Name: sname, + Subjects: []string{subj}, + Replicas: 3, + }) + require_NoError(t, err) + + // Now consumers. + for c := 0; c < numConsumers; c++ { + _, err = js.AddConsumer(sname, &nats.ConsumerConfig{ + Durable: fmt.Sprintf("C-%d", c), + FilterSubject: subj, + AckPolicy: nats.AckExplicitPolicy, + Replicas: 1, + }) + require_NoError(t, err) + } + }() + } + wg.Wait() + + s := c.leader() + js := s.getJetStream() + n := js.getMetaGroup() + // Now let's see how long it takes to create a meta snapshot and how big it is. + start := time.Now() + snap := js.metaSnapshot() + require_NoError(t, n.InstallSnapshot(snap)) + t.Logf("Took %v to snap meta with size of %v\n", time.Since(start), friendlyBytes(len(snap))) +} diff --git a/server/ocsp_responsecache.go b/server/ocsp_responsecache.go index 455fdd3a270..b62166f25d4 100644 --- a/server/ocsp_responsecache.go +++ b/server/ocsp_responsecache.go @@ -1,4 +1,4 @@ -// Copyright 2023 The NATS Authors +// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,7 +15,6 @@ package server import ( "bytes" - "encoding/json" "errors" "fmt" "io" @@ -27,6 +26,7 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" "golang.org/x/crypto/ocsp" diff --git a/server/opts_test.go b/server/opts_test.go index eb1fdcce490..9ae32043fda 100644 --- a/server/opts_test.go +++ b/server/opts_test.go @@ -1,4 +1,4 @@ -// Copyright 2012-2020 The NATS Authors +// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,7 +16,6 @@ package server import ( "bytes" "crypto/tls" - "encoding/json" "flag" "fmt" "net/url" @@ -29,6 +28,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/reload_test.go b/server/reload_test.go index 776f149346e..8c3f3f8c801 100644 --- a/server/reload_test.go +++ b/server/reload_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-2022 The NATS Authors +// Copyright 2017-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,7 +17,6 @@ import ( "bytes" "crypto/tls" "encoding/base64" - "encoding/json" "flag" "fmt" "io" @@ -35,6 +34,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/route.go b/server/route.go index 3ecedf2f11c..5090f7cbf70 100644 --- a/server/route.go +++ b/server/route.go @@ -16,7 +16,6 @@ package server import ( "bytes" "crypto/tls" - "encoding/json" "fmt" "math/rand" "net" @@ -28,6 +27,7 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" ) diff --git a/server/routes_test.go b/server/routes_test.go index 6ff1cdb83b7..671a5e857c5 100644 --- a/server/routes_test.go +++ b/server/routes_test.go @@ -1,4 +1,4 @@ -// Copyright 2013-2023 The NATS Authors +// Copyright 2013-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,7 +18,6 @@ import ( "bytes" "context" "crypto/tls" - "encoding/json" "fmt" "math/rand" "net" @@ -35,6 +34,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" diff --git a/server/server.go b/server/server.go index 242724db161..27ef12477ec 100644 --- a/server/server.go +++ b/server/server.go @@ -17,7 +17,6 @@ import ( "bytes" "context" "crypto/tls" - "encoding/json" "errors" "flag" "fmt" @@ -28,27 +27,27 @@ import ( "net" "net/http" "net/url" - "regexp" - "runtime/pprof" - - // Allow dynamic profiling. - _ "net/http/pprof" "os" "path" "path/filepath" + "regexp" "runtime" + "runtime/pprof" "strconv" "strings" "sync" "sync/atomic" "time" + // Allow dynamic profiling. + _ "net/http/pprof" + + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats-server/v2/logger" "github.com/nats-io/nkeys" "github.com/nats-io/nuid" - - "github.com/nats-io/nats-server/v2/logger" ) const ( diff --git a/server/server_test.go b/server/server_test.go index 0f46b64216a..b7ccb987f72 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -18,7 +18,6 @@ import ( "bytes" "context" "crypto/tls" - "encoding/json" "errors" "flag" "fmt" @@ -35,8 +34,10 @@ import ( "testing" "time" - srvlog "github.com/nats-io/nats-server/v2/logger" + "github.com/goccy/go-json" "github.com/nats-io/nats.go" + + srvlog "github.com/nats-io/nats-server/v2/logger" ) func checkForErr(totalWait, sleepDur time.Duration, f func() error) error { diff --git a/server/stream.go b/server/stream.go index 39c7674bba2..cb82020a1ca 100644 --- a/server/stream.go +++ b/server/stream.go @@ -17,7 +17,6 @@ import ( "archive/tar" "bytes" "encoding/binary" - "encoding/json" "errors" "fmt" "io" @@ -32,6 +31,7 @@ import ( "sync/atomic" "time" + "github.com/goccy/go-json" "github.com/klauspost/compress/s2" "github.com/nats-io/nuid" ) diff --git a/server/tpm/js_ek_tpm_windows.go b/server/tpm/js_ek_tpm_windows.go index 5e401680a08..110675d93c8 100644 --- a/server/tpm/js_ek_tpm_windows.go +++ b/server/tpm/js_ek_tpm_windows.go @@ -17,12 +17,12 @@ package tpm import ( "encoding/base64" - "encoding/json" "fmt" "io" "os" "path/filepath" + "github.com/goccy/go-json" "github.com/google/go-tpm/legacy/tpm2" "github.com/google/go-tpm/tpmutil" "github.com/nats-io/nkeys" diff --git a/server/util.go b/server/util.go index 250fdb3409d..60a4eb90ecd 100644 --- a/server/util.go +++ b/server/util.go @@ -1,4 +1,4 @@ -// Copyright 2012-2019 The NATS Authors +// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,7 +16,6 @@ package server import ( "bytes" "context" - "encoding/json" "errors" "fmt" "math" @@ -26,6 +25,8 @@ import ( "strconv" "strings" "time" + + "github.com/goccy/go-json" ) // This map is used to store URLs string as the key with a reference count as diff --git a/server/websocket_test.go b/server/websocket_test.go index 5cd26c7f8ce..3c9ea72a446 100644 --- a/server/websocket_test.go +++ b/server/websocket_test.go @@ -19,7 +19,6 @@ import ( "crypto/tls" "encoding/base64" "encoding/binary" - "encoding/json" "errors" "fmt" "io" @@ -34,11 +33,11 @@ import ( "testing" "time" + "github.com/goccy/go-json" + "github.com/klauspost/compress/flate" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" - - "github.com/klauspost/compress/flate" ) type testReader struct { diff --git a/test/auth_test.go b/test/auth_test.go index 232ae717769..4d634a51c24 100644 --- a/test/auth_test.go +++ b/test/auth_test.go @@ -1,4 +1,4 @@ -// Copyright 2012-2019 The NATS Authors +// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,12 +14,12 @@ package test import ( - "encoding/json" "fmt" "net" "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/server" ) diff --git a/test/gateway_test.go b/test/gateway_test.go index 49e7274c6c7..bbaad336089 100644 --- a/test/gateway_test.go +++ b/test/gateway_test.go @@ -1,4 +1,4 @@ -// Copyright 2018-2019 The NATS Authors +// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,7 +17,6 @@ import ( "bufio" "bytes" "crypto/tls" - "encoding/json" "fmt" "net" "net/url" @@ -25,6 +24,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) diff --git a/test/leafnode_test.go b/test/leafnode_test.go index 3fb01902bab..5f2bf57be4a 100644 --- a/test/leafnode_test.go +++ b/test/leafnode_test.go @@ -17,7 +17,6 @@ import ( "bytes" "crypto/tls" "crypto/x509" - "encoding/json" "fmt" "math/rand" "net" @@ -30,6 +29,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/logger" "github.com/nats-io/nats-server/v2/server" diff --git a/test/monitor_test.go b/test/monitor_test.go index 91d2f4f07db..661d8e475d7 100644 --- a/test/monitor_test.go +++ b/test/monitor_test.go @@ -1,4 +1,4 @@ -// Copyright 2012-2020 The NATS Authors +// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,7 +16,6 @@ package test import ( "crypto/tls" "crypto/x509" - "encoding/json" "fmt" "io" "net" @@ -28,6 +27,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) diff --git a/test/new_routes_test.go b/test/new_routes_test.go index c4e5b23d1e9..f97758f641c 100644 --- a/test/new_routes_test.go +++ b/test/new_routes_test.go @@ -1,4 +1,4 @@ -// Copyright 2018-2022 The NATS Authors +// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,12 +14,12 @@ package test import ( - "encoding/json" "fmt" "net" "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/logger" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" diff --git a/test/norace_test.go b/test/norace_test.go index eb5252e2a63..d78766efe87 100644 --- a/test/norace_test.go +++ b/test/norace_test.go @@ -19,7 +19,6 @@ package test import ( "context" crand "crypto/rand" - "encoding/json" "fmt" "net" "net/url" @@ -31,6 +30,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nuid" diff --git a/test/ocsp_peer_test.go b/test/ocsp_peer_test.go index ad680f6ff06..0ee51b93805 100644 --- a/test/ocsp_peer_test.go +++ b/test/ocsp_peer_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The NATS Authors +// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,7 +17,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "encoding/json" "errors" "fmt" "io" @@ -27,6 +26,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" . "github.com/nats-io/nats-server/v2/internal/ocsp" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" @@ -3013,12 +3013,12 @@ func TestOCSPMonitoringPort(t *testing.T) { net: 127.0.0.1 port: -1 https: -1 - ocsp { + ocsp { mode = always url = http://127.0.0.1:18888 } store_dir = %s - + tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" diff --git a/test/ports_test.go b/test/ports_test.go index 68b05bbf4c7..2959c00af29 100644 --- a/test/ports_test.go +++ b/test/ports_test.go @@ -1,4 +1,4 @@ -// Copyright 2018-2019 The NATS Authors +// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,7 +14,6 @@ package test import ( - "encoding/json" "errors" "fmt" "os" @@ -23,6 +22,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/server" ) diff --git a/test/proto_test.go b/test/proto_test.go index a03a4446ebc..1ab2df6e538 100644 --- a/test/proto_test.go +++ b/test/proto_test.go @@ -1,4 +1,4 @@ -// Copyright 2012-2020 The NATS Authors +// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,10 +14,10 @@ package test import ( - "encoding/json" "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/server" ) diff --git a/test/route_discovery_test.go b/test/route_discovery_test.go index 3c7a87da259..9c77790e431 100644 --- a/test/route_discovery_test.go +++ b/test/route_discovery_test.go @@ -1,4 +1,4 @@ -// Copyright 2015-2019 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -15,7 +15,6 @@ package test import ( "bufio" - "encoding/json" "fmt" "io" "net" @@ -26,6 +25,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/server" ) diff --git a/test/routes_test.go b/test/routes_test.go index 16254f06c83..76180a63ffd 100644 --- a/test/routes_test.go +++ b/test/routes_test.go @@ -1,4 +1,4 @@ -// Copyright 2012-2019 The NATS Authors +// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,7 +14,6 @@ package test import ( - "encoding/json" "fmt" "io" "net" @@ -25,6 +24,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/internal/testhelper" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" diff --git a/test/service_latency_test.go b/test/service_latency_test.go index ca9c35ede67..6c7e167a7f8 100644 --- a/test/service_latency_test.go +++ b/test/service_latency_test.go @@ -1,4 +1,4 @@ -// Copyright 2019-2021 The NATS Authors +// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,7 +14,6 @@ package test import ( - "encoding/json" "fmt" "math/rand" "net/http" @@ -26,6 +25,7 @@ import ( "testing" "time" + "github.com/goccy/go-json" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" diff --git a/test/test.go b/test/test.go index 52cb873ca17..97e3202f9b3 100644 --- a/test/test.go +++ b/test/test.go @@ -1,4 +1,4 @@ -// Copyright 2012-2019 The NATS Authors +// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,7 +16,6 @@ package test import ( "crypto/rand" "encoding/hex" - "encoding/json" "fmt" "io" "net" @@ -28,8 +27,10 @@ import ( "testing" "time" - srvlog "github.com/nats-io/nats-server/v2/logger" + "github.com/goccy/go-json" "github.com/nats-io/nats-server/v2/server" + + srvlog "github.com/nats-io/nats-server/v2/logger" ) // So we can pass tests and benchmarks.. From f13493af15ad6d71b32303c2cceca2ab3128df44 Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Sat, 23 Nov 2024 18:45:31 -0600 Subject: [PATCH 2/3] Fix for test that depended on stdlib json decode to wrong type to work. Signed-off-by: Derek Collison --- server/norace_test.go | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/server/norace_test.go b/server/norace_test.go index e8f449a780f..0569635989e 100644 --- a/server/norace_test.go +++ b/server/norace_test.go @@ -5371,7 +5371,7 @@ func TestNoRaceJetStreamClusterStreamNamesAndInfosMoreThanAPILimit(t *testing.T) createStream(name) } - // Not using the JS API here beacause we want to make sure that the + // Not using the JS API here because we want to make sure that the // server returns the proper Total count, but also that it does not // send more than when the API limit is in one go. check := func(subj string, limit int) { @@ -5379,19 +5379,35 @@ func TestNoRaceJetStreamClusterStreamNamesAndInfosMoreThanAPILimit(t *testing.T) nreq := JSApiStreamNamesRequest{} b, _ := json.Marshal(nreq) + msg, err := nc.Request(subj, b, 2*time.Second) require_NoError(t, err) - nresp := JSApiStreamNamesResponse{} - json.Unmarshal(msg.Data, &nresp) - if n := nresp.ApiPaged.Total; n != max { - t.Fatalf("Expected total to be %v, got %v", max, n) - } - if n := nresp.ApiPaged.Limit; n != limit { - t.Fatalf("Expected limit to be %v, got %v", limit, n) - } - if n := len(nresp.Streams); n != limit { - t.Fatalf("Expected number of streams to be %v, got %v", limit, n) + switch subj { + case JSApiStreams: + nresp := JSApiStreamNamesResponse{} + json.Unmarshal(msg.Data, &nresp) + if n := nresp.ApiPaged.Total; n != max { + t.Fatalf("Expected total to be %v, got %v", max, n) + } + if n := nresp.ApiPaged.Limit; n != limit { + t.Fatalf("Expected limit to be %v, got %v", limit, n) + } + if n := len(nresp.Streams); n != limit { + t.Fatalf("Expected number of streams to be %v, got %v", limit, n) + } + case JSApiStreamList: + nresp := JSApiStreamListResponse{} + json.Unmarshal(msg.Data, &nresp) + if n := nresp.ApiPaged.Total; n != max { + t.Fatalf("Expected total to be %v, got %v", max, n) + } + if n := nresp.ApiPaged.Limit; n != limit { + t.Fatalf("Expected limit to be %v, got %v", limit, n) + } + if n := len(nresp.Streams); n != limit { + t.Fatalf("Expected number of streams to be %v, got %v", limit, n) + } } } From 2795eca16c2370391bca0d89fec381285756c503 Mon Sep 17 00:00:00 2001 From: Derek Collison Date: Sun, 24 Nov 2024 10:07:24 -0600 Subject: [PATCH 3/3] Increase minimum interval for meta snapshots and do not pre-empt on consumer removals to avoind excesive snapshotting. Signed-off-by: Derek Collison --- server/jetstream_cluster.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server/jetstream_cluster.go b/server/jetstream_cluster.go index bdd364cf5ce..15cf5809d44 100644 --- a/server/jetstream_cluster.go +++ b/server/jetstream_cluster.go @@ -1320,7 +1320,7 @@ func (js *jetStream) monitorCluster() { isLeader bool lastSnapTime time.Time compactSizeMin = uint64(8 * 1024 * 1024) // 8MB - minSnapDelta = 10 * time.Second + minSnapDelta = 30 * time.Second ) // Highwayhash key for generating hashes. @@ -1410,7 +1410,7 @@ func (js *jetStream) monitorCluster() { go checkHealth() continue } - if didSnap, didStreamRemoval, didConsumerRemoval, err := js.applyMetaEntries(ce.Entries, ru); err == nil { + if didSnap, didStreamRemoval, _, err := js.applyMetaEntries(ce.Entries, ru); err == nil { var nb uint64 // Some entries can fail without an error when shutting down, don't move applied forward. if !js.isShuttingDown() { @@ -1418,8 +1418,6 @@ func (js *jetStream) monitorCluster() { } if js.hasPeerEntries(ce.Entries) || didStreamRemoval || (didSnap && !isLeader) { doSnapshot() - } else if didConsumerRemoval && time.Since(lastSnapTime) > minSnapDelta/2 { - doSnapshot() } else if nb > compactSizeMin && time.Since(lastSnapTime) > minSnapDelta { doSnapshot() }