feat: export Prom metrics for logs. Fixes #418

argoproj-labs · Oct 5, 2021 · 29b4deb · 29b4deb
1 parent 64f8f10
commit 29b4deb
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 23 deletions.
diff --git a/go.mod b/go.mod
@@ -45,6 +45,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/uber/jaeger-client-go v2.29.1+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
+	github.com/weaveworks/promrus v1.2.0
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.16.0 // indirect
 	golang.org/x/crypto v0.0.0-20210915214749-c084706c2272 // indirect

diff --git a/go.sum b/go.sum
@@ -598,6 +598,8 @@ github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
+github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M=
+github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA=
 github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=

diff --git a/shared/util/log.go b/shared/util/log.go
@@ -1,33 +1,33 @@
 package util
 
 import (
+	"bytes"
+	"os"
+
 	"github.com/bombsimon/logrusr"
 	"github.com/go-logr/logr"
 	log "github.com/sirupsen/logrus"
-	"github.com/sirupsen/logrus/hooks/writer"
-	"io"
-
-	"os"
+	"github.com/weaveworks/promrus"
 )
 
-func NewLogger() logr.Logger {
+type splitter int
+
+func (splitter) Write(p []byte) (n int, err error) {
+	if bytes.Contains(p, []byte("level=error")) {
+		return os.Stderr.Write(p)
+	}
+	return os.Stdout.Write(p)
+}
+
+var logger = newLogger()
+
+func newLogger() logr.Logger {
 	l := log.New()
-	l.SetOutput(io.Discard)
-	l.AddHook(&writer.Hook{
-		Writer: os.Stderr,
-		LogLevels: []log.Level{
-			log.PanicLevel,
-			log.FatalLevel,
-			log.ErrorLevel,
-			log.WarnLevel,
-		},
-	})
-	l.AddHook(&writer.Hook{
-		Writer: os.Stdout,
-		LogLevels: []log.Level{
-			log.InfoLevel,
-			log.DebugLevel,
-		},
-	})
+	l.SetOutput(splitter(0))
+	l.AddHook(promrus.MustNewPrometheusHook())
 	return logrusr.NewLogger(l)
 }
+
+func NewLogger() logr.Logger {
+	return logger
+}
diff --git a/shared/util/version.go b/shared/util/version.go
@@ -10,7 +10,6 @@ var (
 	// * "v0.0.0-X-Y" for unreleased versions, e.g.
 	//   - "v0.0.0-latest-0" ("latest" version, i.e. latest build on the "main" branch or local dev build)
 	version = "v0.0.0-latest-0"
-	logger  = NewLogger()
 	Version semver.Version
 )
 

diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go
@@ -57,6 +57,7 @@ func TestMetrics(t *testing.T) {
 	ExpectMetric("sinks_errors", Eq(3))
 	sendHTTPMsag("my-msg")
 	ExpectMetric("sources_totalBytes", Eq(12))
+	ExpectMetric("log_messages", Gt(10))
 }
 
 func sendHTTPMsag(msg string) {