diff --git a/go.mod b/go.mod index 51937ac2..35b0ec37 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,10 @@ go 1.25.5 require ( github.com/antonfisher/nested-logrus-formatter v1.3.1 github.com/fsnotify/fsnotify v1.9.0 - github.com/klauspost/compress v1.18.0 + github.com/klauspost/compress v1.18.3 github.com/nats-io/jsm.go v0.2.4 - github.com/nats-io/nats-server/v2 v2.11.9 - github.com/nats-io/nats.go v1.46.0 + github.com/nats-io/nats-server/v2 v2.12.3 + github.com/nats-io/nats.go v1.47.0 github.com/nats-io/nuid v1.0.1 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 @@ -19,25 +19,25 @@ require ( github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/thediveo/enumflag/v2 v2.0.7 - golang.org/x/crypto v0.45.0 - golang.org/x/sync v0.18.0 + golang.org/x/crypto v0.46.0 + golang.org/x/sync v0.19.0 ) require ( - github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op // indirect + github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/expr-lang/expr v1.17.7 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect - github.com/google/go-tpm v0.9.5 // indirect + github.com/google/go-tpm v0.9.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/minio/highwayhash v1.0.3 // indirect + github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nats-io/jwt/v2 v2.7.4 // indirect - github.com/nats-io/nkeys v0.4.11 // indirect + github.com/nats-io/jwt/v2 v2.8.0 // indirect + github.com/nats-io/nkeys v0.4.12 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.17.0 // indirect @@ -49,9 +49,9 @@ require ( go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20250717185816-542afb5b7346 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect - golang.org/x/time v0.13.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/time v0.14.0 // indirect google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 01a16f45..b1b07a5b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op h1:+OSa/t11TFhqfrX0EOSqQBDJ0YlpmK0rDSiB19dg9M0= -github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E= +github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op h1:Ucf+QxEKMbPogRO5guBNe5cgd9uZgfoJLOYs8WWhtjM= +github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E= github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UMEoHck02Q9L0FP13b/xSbQ= github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -27,34 +27,34 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= -github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm v0.9.7 h1:u89J4tUUeDTlH8xxC3CTW7OHZjbjKoHdQ9W7gCUhtxA= +github.com/google/go-tpm v0.9.7/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q= -github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk= +github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nats-io/jsm.go v0.2.4 h1:3cvx6D8rYYbdKe7U/zi6Cs62EoWPL/vuBsisZqn1/2Y= github.com/nats-io/jsm.go v0.2.4/go.mod h1:uzB1RF15TpzlFVM7d2AYaGUiyAZKTDjIiJ12kPL52gE= -github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI= -github.com/nats-io/jwt/v2 v2.7.4/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA= -github.com/nats-io/nats-server/v2 v2.11.9 h1:k7nzHZjUf51W1b08xiQih63Rdxh0yr5O4K892Mx5gQA= -github.com/nats-io/nats-server/v2 v2.11.9/go.mod h1:1MQgsAQX1tVjpf3Yzrk3x2pzdsZiNL/TVP3Amhp3CR8= -github.com/nats-io/nats.go v1.46.0 h1:iUcX+MLT0HHXskGkz+Sg20sXrPtJLsOojMDTDzOHSb8= -github.com/nats-io/nats.go v1.46.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= -github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= -github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g= +github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA= +github.com/nats-io/nats-server/v2 v2.12.3 h1:KRv+1n7lddMVgkJPQer+pt36TcO0ENxjilBmeWdjcHs= +github.com/nats-io/nats-server/v2 v2.12.3/go.mod h1:MQXjG9WjyXKz9koWzUc3jYUMKD8x3CLmTNy91IQQz3Y= +github.com/nats-io/nats.go v1.47.0 h1:YQdADw6J/UfGUd2Oy6tn4Hq6YHxCaJrVKayxxFqYrgM= +github.com/nats-io/nats.go v1.47.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= +github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc= +github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= @@ -109,24 +109,24 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20250717185816-542afb5b7346 h1:vuCObX8mQzik1tfEcYxWZBuVsmQtD1IjxCyPKM18Bh4= golang.org/x/exp v0.0.0-20250717185816-542afb5b7346/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= -golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/surveyor/collector_statz.go b/surveyor/collector_statz.go index ed8cf88b..c9708d15 100644 --- a/surveyor/collector_statz.go +++ b/surveyor/collector_statz.go @@ -108,6 +108,10 @@ type statzDescs struct { JetstreamClusterRaftGroupReplicaActive *GaugeVec JetstreamClusterRaftGroupReplicaCurrent *GaugeVec JetstreamClusterRaftGroupReplicaOffline *GaugeVec + // JetStream meta snapshot stats + JetstreamMetaSnapshotPendingEntries *GaugeVec + JetstreamMetaSnapshotPendingBytes *GaugeVec + JetstreamMetaSnapshotLastDuration *GaugeVec // JetStream server stats JetstreamServerDisabled *GaugeVec JetstreamServerStreams *GaugeVec @@ -341,6 +345,22 @@ func (sc *StatzCollector) gatewayLabelValues(sm *server.ServerInfo, gStat *serve return []string{sm.Cluster, serverName(sm), sm.ID, gStat.Name, strconv.FormatUint(gStat.ID, 10)} } +// safeMsgBytesBytes returns the Bytes value from a MsgBytes pointer, or 0 if nil +func safeMsgBytesBytes(mb *server.MsgBytes) int64 { + if mb == nil { + return 0 + } + return mb.Bytes +} + +// safeMsgBytesMsgs returns the Msgs value from a MsgBytes pointer, or 0 if nil +func safeMsgBytesMsgs(mb *server.MsgBytes) int64 { + if mb == nil { + return 0 + } + return mb.Msgs +} + // Up/Down on servers - look at discovery mechanisms in Prometheus - aging out, how does it work? func (sc *StatzCollector) buildDescs() { newName := func(name string) string { @@ -424,6 +444,12 @@ func (sc *StatzCollector) buildDescs() { sc.descs.JetstreamClusterRaftGroupReplicaCurrent = newGaugeVec(newName("jetstream_cluster_raft_group_replica_peer_current"), "Jetstream RAFT Group Peer is current: 1 or not: 0", sc.constLabels, jsClusterReplicaLabelKeys) sc.descs.JetstreamClusterRaftGroupReplicaOffline = newGaugeVec(newName("jetstream_cluster_raft_group_replica_peer_offline"), "Jetstream RAFT Group Peer is offline: 1 or online: 0", sc.constLabels, jsClusterReplicaLabelKeys) + // Meta snapshot stats + jsMetaSnapshotLabelKeys := []string{"server_id", "server_name", "cluster_name"} + sc.descs.JetstreamMetaSnapshotPendingEntries = newGaugeVec(newName("jetstream_meta_snapshot_pending_entries"), "Number of pending entries awaiting meta snapshot", sc.constLabels, jsMetaSnapshotLabelKeys) + sc.descs.JetstreamMetaSnapshotPendingBytes = newGaugeVec(newName("jetstream_meta_snapshot_pending_bytes"), "Size in bytes of pending entries awaiting meta snapshot", sc.constLabels, jsMetaSnapshotLabelKeys) + sc.descs.JetstreamMetaSnapshotLastDuration = newGaugeVec(newName("jetstream_meta_snapshot_last_duration"), "Duration of the last meta snapshot in nanoseconds", sc.constLabels, jsMetaSnapshotLabelKeys) + jsServerLabelKeys := []string{"server_id", "server_name", "cluster_name"} sc.descs.JetstreamServerDisabled = newGaugeVec(newName("jetstream_server_jetstream_disabled"), "JetStream disabled or not", sc.constLabels, jsServerLabelKeys) sc.descs.JetstreamServerStreams = newGaugeVec(newName("jetstream_server_total_streams"), "Total number of streams in JetStream", sc.constLabels, jsServerLabelKeys) @@ -1430,6 +1456,10 @@ func (sc *StatzCollector) MetricInfos() []MetricInfo { sc.descs.JetstreamClusterRaftGroupReplicaActive, sc.descs.JetstreamClusterRaftGroupReplicaCurrent, sc.descs.JetstreamClusterRaftGroupReplicaOffline, + // JetStream meta snapshot stats + sc.descs.JetstreamMetaSnapshotPendingEntries, + sc.descs.JetstreamMetaSnapshotPendingBytes, + sc.descs.JetstreamMetaSnapshotLastDuration, } // Account scope metrics @@ -1687,6 +1717,15 @@ func (sc *StatzCollector) Collect(ch chan<- prometheus.Metric) { metrics.newGaugeMetric(sc.descs.JetstreamClusterRaftGroupReplicaOffline, float64(0), jsClusterReplicaLabelValues) } } + + // Meta snapshot stats + if sm.Stats.JetStream.Meta.Snapshot != nil { + jsMetaSnapshotLabelValues := []string{sm.Server.ID, serverName(&sm.Server), sm.Server.Cluster} + snapshot := sm.Stats.JetStream.Meta.Snapshot + metrics.newGaugeMetric(sc.descs.JetstreamMetaSnapshotPendingEntries, float64(snapshot.PendingEntries), jsMetaSnapshotLabelValues) + metrics.newGaugeMetric(sc.descs.JetstreamMetaSnapshotPendingBytes, float64(snapshot.PendingSize), jsMetaSnapshotLabelValues) + metrics.newGaugeMetric(sc.descs.JetstreamMetaSnapshotLastDuration, float64(snapshot.LastDuration), jsMetaSnapshotLabelValues) + } } } @@ -1728,22 +1767,22 @@ func (sc *StatzCollector) Collect(ch chan<- prometheus.Metric) { metrics.newCounterMetric(sc.descs.accMsgsRecv, float64(as.Data.Received.Msgs), serverAndAccLabels) if sc.collectAccountsDetailed { - metrics.newCounterMetric(sc.descs.accClientBytesSent, float64(as.Data.Sent.Bytes-as.Data.Sent.Routes.Bytes-as.Data.Sent.Gateways.Bytes-as.Data.Sent.Leafs.Bytes), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accClientBytesRecv, float64(as.Data.Received.Bytes-as.Data.Received.Routes.Bytes-as.Data.Received.Gateways.Bytes-as.Data.Received.Leafs.Bytes), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accClientMsgsSent, float64(as.Data.Sent.Msgs-as.Data.Sent.Routes.Msgs-as.Data.Sent.Gateways.Msgs-as.Data.Sent.Leafs.Msgs), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accClientMsgsRecv, float64(as.Data.Received.Msgs-as.Data.Received.Routes.Msgs-as.Data.Received.Gateways.Msgs-as.Data.Received.Leafs.Msgs), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accLeafBytesSent, float64(as.Data.Sent.Leafs.Bytes), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accLeafBytesRecv, float64(as.Data.Received.Leafs.Bytes), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accLeafMsgsSent, float64(as.Data.Sent.Leafs.Msgs), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accLeafMsgsRecv, float64(as.Data.Received.Leafs.Msgs), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accRouteBytesSent, float64(as.Data.Sent.Routes.Bytes), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accRouteBytesRecv, float64(as.Data.Received.Routes.Bytes), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accRouteMsgsSent, float64(as.Data.Sent.Routes.Msgs), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accRouteMsgsRecv, float64(as.Data.Received.Routes.Msgs), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accGatewayBytesSent, float64(as.Data.Sent.Gateways.Bytes), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accGatewayBytesRecv, float64(as.Data.Received.Gateways.Bytes), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accGatewayMsgsSent, float64(as.Data.Sent.Gateways.Msgs), serverAndAccLabels) - metrics.newCounterMetric(sc.descs.accGatewayMsgsRecv, float64(as.Data.Received.Gateways.Msgs), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accClientBytesSent, float64(as.Data.Sent.Bytes-safeMsgBytesBytes(as.Data.Sent.Routes)-safeMsgBytesBytes(as.Data.Sent.Gateways)-safeMsgBytesBytes(as.Data.Sent.Leafs)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accClientBytesRecv, float64(as.Data.Received.Bytes-safeMsgBytesBytes(as.Data.Received.Routes)-safeMsgBytesBytes(as.Data.Received.Gateways)-safeMsgBytesBytes(as.Data.Received.Leafs)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accClientMsgsSent, float64(as.Data.Sent.Msgs-safeMsgBytesMsgs(as.Data.Sent.Routes)-safeMsgBytesMsgs(as.Data.Sent.Gateways)-safeMsgBytesMsgs(as.Data.Sent.Leafs)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accClientMsgsRecv, float64(as.Data.Received.Msgs-safeMsgBytesMsgs(as.Data.Received.Routes)-safeMsgBytesMsgs(as.Data.Received.Gateways)-safeMsgBytesMsgs(as.Data.Received.Leafs)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accLeafBytesSent, float64(safeMsgBytesBytes(as.Data.Sent.Leafs)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accLeafBytesRecv, float64(safeMsgBytesBytes(as.Data.Received.Leafs)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accLeafMsgsSent, float64(safeMsgBytesMsgs(as.Data.Sent.Leafs)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accLeafMsgsRecv, float64(safeMsgBytesMsgs(as.Data.Received.Leafs)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accRouteBytesSent, float64(safeMsgBytesBytes(as.Data.Sent.Routes)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accRouteBytesRecv, float64(safeMsgBytesBytes(as.Data.Received.Routes)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accRouteMsgsSent, float64(safeMsgBytesMsgs(as.Data.Sent.Routes)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accRouteMsgsRecv, float64(safeMsgBytesMsgs(as.Data.Received.Routes)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accGatewayBytesSent, float64(safeMsgBytesBytes(as.Data.Sent.Gateways)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accGatewayBytesRecv, float64(safeMsgBytesBytes(as.Data.Received.Gateways)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accGatewayMsgsSent, float64(safeMsgBytesMsgs(as.Data.Sent.Gateways)), serverAndAccLabels) + metrics.newCounterMetric(sc.descs.accGatewayMsgsRecv, float64(safeMsgBytesMsgs(as.Data.Received.Gateways)), serverAndAccLabels) } } diff --git a/surveyor/collector_statz_test.go b/surveyor/collector_statz_test.go index 2c5655e3..7575524f 100644 --- a/surveyor/collector_statz_test.go +++ b/surveyor/collector_statz_test.go @@ -412,6 +412,55 @@ func TestStatzCollector_GoMemLimit(t *testing.T) { } } +func TestStatzCollector_WithStats_JetstreamMetaSnapshot(t *testing.T) { + statsRaw, err := os.ReadFile("testdata/stats/stats-meta-snapshot.json") + if err != nil { + t.Fatalf("error reading testdata: %v", err) + } + stats := &server.ServerStatsMsg{} + err = json.Unmarshal(statsRaw, stats) + if err != nil { + t.Fatalf("error unmarshalling stats: %v", err) + } + + sc, err := NewStatzCollectorOpts( + WithStats(WithStatsBatch{ + Stats: []*server.ServerStatsMsg{stats}, + }), + ) + if err != nil { + t.Fatalf("error creating statz collector: %v", err) + } + + output := gatherStatzCollectorMetrics(t, sc) + + // Verify all JetStream meta cluster snapshot metrics are present + want := []string{ + "nats_core_jetstream_meta_snapshot_pending_entries", + "nats_core_jetstream_meta_snapshot_pending_bytes", + "nats_core_jetstream_meta_snapshot_last_duration", + } + + for _, m := range want { + if !strings.Contains(output, m) { + t.Fatalf("invalid output, missing '%s':\n%v\n", m, output) + } + } + + // Verify specific metric values + expectedValues := []string{ + "nats_core_jetstream_meta_snapshot_pending_entries{cluster_name=\"meta-cluster\",server_id=\"meta-server1\",server_name=\"meta-server1\"} 1500", + "nats_core_jetstream_meta_snapshot_pending_bytes{cluster_name=\"meta-cluster\",server_id=\"meta-server1\",server_name=\"meta-server1\"} 524288", + "nats_core_jetstream_meta_snapshot_last_duration{cluster_name=\"meta-cluster\",server_id=\"meta-server1\",server_name=\"meta-server1\"} 1.23456789e+09", + } + + for _, expectedValue := range expectedValues { + if !strings.Contains(output, expectedValue) { + t.Fatalf("expected metric value not found. Expected: %s\nActual output:\n%v", expectedValue, output) + } + } +} + func TestStatzCollector_MetricInfos(t *testing.T) { sc, err := NewStatzCollectorOpts( WithStats(WithStatsBatch{ diff --git a/surveyor/testdata/stats/stats-meta-snapshot.json b/surveyor/testdata/stats/stats-meta-snapshot.json new file mode 100644 index 00000000..17e99f2f --- /dev/null +++ b/surveyor/testdata/stats/stats-meta-snapshot.json @@ -0,0 +1,82 @@ +{ + "server": { + "name": "meta-server1", + "host": "0.0.0.0", + "id": "meta-server1", + "cluster": "meta-cluster", + "version": "2.12.3", + "jetstream": true, + "flags": 1 + }, + "statsz": { + "start": "2026-01-01T00:00:00Z", + "mem": 2048, + "cores": 4, + "cpu": 0.3, + "connections": 20, + "total_connections": 200, + "active_accounts": 15, + "num_subs": 150, + "sent": { + "bytes": 2048, + "msgs": 200 + }, + "received": { + "bytes": 2048, + "msgs": 200 + }, + "active_servers": 3, + "jetstream": { + "config": { + "max_memory": 2048, + "max_storage": 2048, + "sync_interval": 2000, + "sync_always": true + }, + "stats": { + "memory": 2048, + "storage": 2048, + "reserved_memory": 1024, + "reserved_storage": 1024, + "accounts": 15, + "ha_assets": 15, + "api": { + "level": 1, + "total": 100, + "errors": 2 + } + }, + "meta": { + "name": "meta-cluster", + "leader": "meta-server1", + "peer": "meta-server2", + "replicas": [ + { + "name": "meta-server2", + "current": true, + "offline": false, + "active": 2000, + "lag": 50, + "peer": "meta-server2" + }, + { + "name": "meta-server3", + "current": true, + "offline": false, + "active": 2100, + "lag": 25, + "peer": "meta-server3" + } + ], + "size": 3, + "pending": 0, + "snapshot": { + "pending_entries": 1500, + "pending_size": 524288, + "last_time": "2026-01-20T15:30:45.123456789Z", + "last_duration": 1234567890 + } + } + } + } +}