From 78f8ec7e2e05432e40010cde4c800a9c7fa81b5d Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Tue, 30 Jul 2019 14:53:16 +0100 Subject: [PATCH] Expose the number of open connections. Currently you'd have to subtract counters, which is not advised as it's racy. --- dialer_reporter.go | 12 ++++++++++++ dialer_test.go | 9 +++++++++ listener_reporter.go | 11 +++++++++++ listener_test.go | 7 +++++++ 4 files changed, 39 insertions(+) diff --git a/dialer_reporter.go b/dialer_reporter.go index 0e39886..c7b3c10 100644 --- a/dialer_reporter.go +++ b/dialer_reporter.go @@ -53,6 +53,14 @@ var ( Name: "dialer_conn_closed_total", Help: "Total number of connections closed which originated from the dialer of a given name.", }, []string{"dialer_name"}) + + dialerConnOpen = prom.NewGaugeVec( + prom.GaugeOpts{ + Namespace: "net", + Subsystem: "conntrack", + Name: "dialer_conn_open", + Help: "Number of open connections which originated from the dialer of a given name.", + }, []string{"dialer_name"}) ) func init() { @@ -60,6 +68,7 @@ func init() { prom.MustRegister(dialerConnEstablishedTotal) prom.MustRegister(dialerConnFailedTotal) prom.MustRegister(dialerConnClosedTotal) + prom.MustRegister(dialerConnOpen) } // preRegisterDialerMetrics pre-populates Prometheus labels for the given dialer name, to avoid Prometheus missing labels issue. @@ -70,6 +79,7 @@ func PreRegisterDialerMetrics(dialerName string) { dialerConnFailedTotal.WithLabelValues(dialerName, string(reason)) } dialerConnClosedTotal.WithLabelValues(dialerName) + dialerConnOpen.WithLabelValues(dialerName) } func reportDialerConnAttempt(dialerName string) { @@ -78,10 +88,12 @@ func reportDialerConnAttempt(dialerName string) { func reportDialerConnEstablished(dialerName string) { dialerConnEstablishedTotal.WithLabelValues(dialerName).Inc() + dialerConnOpen.WithLabelValues(dialerName).Inc() } func reportDialerConnClosed(dialerName string) { dialerConnClosedTotal.WithLabelValues(dialerName).Inc() + dialerConnOpen.WithLabelValues(dialerName).Dec() } func reportDialerConnFailed(dialerName string, err error) { diff --git a/dialer_test.go b/dialer_test.go index 0471029..3df5603 100644 --- a/dialer_test.go +++ b/dialer_test.go @@ -62,6 +62,9 @@ func (s *DialerTestSuite) TestDialerMetricsArePreregistered() { {"net_conntrack_dialer_conn_failed_total", []string{"default", "refused"}}, {"net_conntrack_dialer_conn_failed_total", []string{"default", "timeout"}}, {"net_conntrack_dialer_conn_failed_total", []string{"default", "unknown"}}, + {"net_conntrack_dialer_conn_open", []string{"default"}}, + {"net_conntrack_dialer_conn_open", []string{"foobar"}}, + {"net_conntrack_dialer_conn_open", []string{"something_manual"}}, } { lineCount := len(fetchPrometheusLines(s.T(), testCase.metricName, testCase.existingLabels...)) assert.NotEqual(s.T(), 0, lineCount, "metrics must exist for test case %d", testId) @@ -81,6 +84,7 @@ func (s *DialerTestSuite) TestDialerMetricsAreNotPreregisteredWithMonitoringOff( {"net_conntrack_dialer_conn_failed_total", []string{"nomon", "refused"}}, {"net_conntrack_dialer_conn_failed_total", []string{"nomon", "timeout"}}, {"net_conntrack_dialer_conn_failed_total", []string{"nomon", "unknown"}}, + {"net_conntrack_dialer_conn_open", []string{"nomon"}}, } { lineCount := len(fetchPrometheusLines(s.T(), testCase.metricName, testCase.existingLabels...)) assert.Equal(s.T(), 0, lineCount, "metrics should not be registered exist for test case %d", testId) @@ -93,6 +97,7 @@ func (s *DialerTestSuite) TestDialerUnderNormalConnection() { beforeAttempts := sumCountersForMetricAndLabels(s.T(), "net_conntrack_dialer_conn_attempted_total", "normal_conn") beforeEstablished := sumCountersForMetricAndLabels(s.T(), "net_conntrack_dialer_conn_established_total", "normal_conn") beforeClosed := sumCountersForMetricAndLabels(s.T(), "net_conntrack_dialer_conn_closed_total", "normal_conn") + beforeOpen := sumCountersForMetricAndLabels(s.T(), "net_conntrack_dialer_conn_open", "normal_conn") conn, err := dialFunc(context.TODO(), "tcp", s.serverListener.Addr().String()) require.NoError(s.T(), err, "NewDialContextFunc should successfully establish a conn here") @@ -102,9 +107,13 @@ func (s *DialerTestSuite) TestDialerUnderNormalConnection() { "the established conn counter must be incremented after connection was opened") assert.Equal(s.T(), beforeClosed, sumCountersForMetricAndLabels(s.T(), "net_conntrack_dialer_conn_closed_total", "normal_conn"), "the closed conn counter must not be incremented after connection was opened") + assert.Equal(s.T(), beforeOpen+1, sumCountersForMetricAndLabels(s.T(), "net_conntrack_dialer_conn_open", "normal_conn"), + "the open conn gauge must be incremented after connection was opened") conn.Close() assert.Equal(s.T(), beforeClosed+1, sumCountersForMetricAndLabels(s.T(), "net_conntrack_dialer_conn_established_total", "normal_conn"), "the closed conn counter must be incremented after connection was closed") + assert.Equal(s.T(), beforeOpen, sumCountersForMetricAndLabels(s.T(), "net_conntrack_dialer_conn_open", "normal_conn"), + "the open conn gauge must be decremented after connection was closed") } func (s *DialerTestSuite) TestDialerWithContextName() { diff --git a/listener_reporter.go b/listener_reporter.go index 21a8f55..26ad81d 100644 --- a/listener_reporter.go +++ b/listener_reporter.go @@ -21,23 +21,34 @@ var ( Name: "listener_conn_closed_total", Help: "Total number of connections closed that were made to the listener of a given name.", }, []string{"listener_name"}) + listenerOpen = prom.NewGaugeVec( + prom.GaugeOpts{ + Namespace: "net", + Subsystem: "conntrack", + Name: "listener_conn_open", + Help: "Number of open connections to the listener of a given name.", + }, []string{"listener_name"}) ) func init() { prom.MustRegister(listenerAcceptedTotal) prom.MustRegister(listenerClosedTotal) + prom.MustRegister(listenerOpen) } // preRegisterListener pre-populates Prometheus labels for the given listener name, to avoid Prometheus missing labels issue. func preRegisterListenerMetrics(listenerName string) { listenerAcceptedTotal.WithLabelValues(listenerName) listenerClosedTotal.WithLabelValues(listenerName) + listenerOpen.WithLabelValues(listenerName) } func reportListenerConnAccepted(listenerName string) { listenerAcceptedTotal.WithLabelValues(listenerName).Inc() + listenerOpen.WithLabelValues(listenerName).Inc() } func reportListenerConnClosed(listenerName string) { listenerClosedTotal.WithLabelValues(listenerName).Inc() + listenerOpen.WithLabelValues(listenerName).Dec() } diff --git a/listener_test.go b/listener_test.go index 2bd86b1..4ebc17d 100644 --- a/listener_test.go +++ b/listener_test.go @@ -58,8 +58,10 @@ func (s *ListenerTestSuite) TestTrackingMetricsPreregistered() { }{ {"net_conntrack_listener_conn_accepted_total", []string{"default"}}, {"net_conntrack_listener_conn_closed_total", []string{"default"}}, + {"net_conntrack_listener_conn_open", []string{"default"}}, {"net_conntrack_listener_conn_accepted_total", []string{listenerName}}, {"net_conntrack_listener_conn_closed_total", []string{listenerName}}, + {"net_conntrack_listener_conn_open", []string{listenerName}}, } { lineCount := len(fetchPrometheusLines(s.T(), testCase.metricName, testCase.existingLabels...)) assert.NotEqual(s.T(), 0, lineCount, "metrics must exist for test case %d", testId) @@ -70,6 +72,7 @@ func (s *ListenerTestSuite) TestMonitoringNormalConns() { beforeAccepted := sumCountersForMetricAndLabels(s.T(), "net_conntrack_listener_conn_accepted_total", listenerName) beforeClosed := sumCountersForMetricAndLabels(s.T(), "net_conntrack_listener_conn_closed_total", listenerName) + beforeOpen := sumCountersForMetricAndLabels(s.T(), "net_conntrack_listener_conn_open", listenerName) conn, err := (&net.Dialer{}).DialContext(context.TODO(), "tcp", s.serverListener.Addr().String()) require.NoError(s.T(), err, "DialContext should successfully establish a conn here") @@ -77,9 +80,13 @@ func (s *ListenerTestSuite) TestMonitoringNormalConns() { "the accepted conn counter must be incremented after connection was opened") assert.Equal(s.T(), beforeClosed, sumCountersForMetricAndLabels(s.T(), "net_conntrack_listener_conn_closed_total", listenerName), "the closed conn counter must not be incremented before the connection is closed") + assert.Equal(s.T(), beforeOpen+1, sumCountersForMetricAndLabels(s.T(), "net_conntrack_listener_conn_open", listenerName), + "the open conn must be incremented when the connection is opened") conn.Close() assert.Equal(s.T(), beforeClosed+1, sumCountersForMetricAndLabels(s.T(), "net_conntrack_listener_conn_closed_total", listenerName), "the closed conn counter must be incremented after connection was closed") + assert.Equal(s.T(), beforeOpen, sumCountersForMetricAndLabels(s.T(), "net_conntrack_listener_conn_open", listenerName), + "the open conn must be decremented when the connection is closed") } func (s *ListenerTestSuite) TestTracingNormalComms() {