[9.0](backport #46054) [Chore] Accomodate breaking from elastic-agent-libs and elastic-agent-system-metrics

NOTICE.txt

@@ -12350,11 +12350,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.21.5
+Version: v0.23.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.21.5/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.23.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -12561,11 +12561,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.11.11
+Version: v0.13.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.11.11/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.13.2/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004

filebeat/input/filestream/internal/input-logfile/metrics.go

@@ -21,6 +21,7 @@ import (
 	"github.com/rcrowley/go-metrics"
 
 	"github.com/elastic/beats/v7/libbeat/monitoring/inputmon"
+	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/monitoring"
 	"github.com/elastic/elastic-agent-libs/monitoring/adapter"
 )
@@ -82,7 +83,7 @@ func NewMetrics(id string) *Metrics {
 		HarvesterRunning:   monitoring.NewInt(harvesterMetrics, "running"),
 		HarvesterOpenFiles: monitoring.NewInt(harvesterMetrics, "open_files"),
 	}
-	_ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept).
+	_ = adapter.NewGoMetrics(reg, "processing_time", logp.NewLogger(""), adapter.Accept).
 		Register("histogram", metrics.NewHistogram(m.ProcessingTime))
 
 	return &m

filebeat/input/kafka/config.go

@@ -209,6 +209,7 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) {
 	k.MetricRegistry = adapter.GetGoMetrics(
 		monitoring.Default,
 		"filebeat.inputs.kafka",
+		logp.NewLogger(""),
 		adapter.Rename("incoming-byte-rate", "bytes_read"),
 		adapter.Rename("outgoing-byte-rate", "bytes_write"),
 		adapter.GoMetricsNilify,

filebeat/input/netmetrics/tcp.go

@@ -67,9 +67,9 @@ func NewTCP(inputName string, id string, device string, poll time.Duration, log
 		arrivalPeriod:   metrics.NewUniformSample(1024),
 		processingTime:  metrics.NewUniformSample(1024),
 	}
-	_ = adapter.NewGoMetrics(reg, "arrival_period", adapter.Accept).
+	_ = adapter.NewGoMetrics(reg, "arrival_period", log, adapter.Accept).
 		Register("histogram", metrics.NewHistogram(out.arrivalPeriod))
-	_ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept).
+	_ = adapter.NewGoMetrics(reg, "processing_time", log, adapter.Accept).
 		Register("histogram", metrics.NewHistogram(out.processingTime))
 
 	out.device.Set(device)

filebeat/input/netmetrics/udp.go

@@ -71,9 +71,9 @@ func NewUDP(inputName string, id string, device string, buflen uint64, poll time
 		arrivalPeriod:   metrics.NewUniformSample(1024),
 		processingTime:  metrics.NewUniformSample(1024),
 	}
-	_ = adapter.NewGoMetrics(reg, "arrival_period", adapter.Accept).
+	_ = adapter.NewGoMetrics(reg, "arrival_period", log, adapter.Accept).
 		Register("histogram", metrics.NewHistogram(out.arrivalPeriod))
-	_ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept).
+	_ = adapter.NewGoMetrics(reg, "processing_time", log, adapter.Accept).
 		Register("histogram", metrics.NewHistogram(out.processingTime))
 
 	out.device.Set(device)

filebeat/input/unix/input.go

@@ -164,9 +164,10 @@ func newInputMetrics(id, path string, log *logp.Logger) *inputMetrics {
 		arrivalPeriod:  metrics.NewUniformSample(1024),
 		processingTime: metrics.NewUniformSample(1024),
 	}
-	_ = adapter.NewGoMetrics(reg, "arrival_period", adapter.Accept).
+	logger := logp.NewLogger("")
+	_ = adapter.NewGoMetrics(reg, "arrival_period", logger, adapter.Accept).
 		Register("histogram", metrics.NewHistogram(out.arrivalPeriod))
-	_ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept).
+	_ = adapter.NewGoMetrics(reg, "processing_time", logger, adapter.Accept).
 		Register("histogram", metrics.NewHistogram(out.processingTime))
 
 	out.path.Set(path)

go.mod

@@ -177,8 +177,8 @@ require (
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/ebpfevents v0.7.0
 	github.com/elastic/elastic-agent-autodiscover v0.9.2
-	github.com/elastic/elastic-agent-libs v0.21.5
-	github.com/elastic/elastic-agent-system-metrics v0.11.11
+	github.com/elastic/elastic-agent-libs v0.23.0
+	github.com/elastic/elastic-agent-system-metrics v0.13.2
 	github.com/elastic/go-elasticsearch/v8 v8.17.1
 	github.com/elastic/go-quark v0.3.0
 	github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727

go.sum

@@ -354,10 +354,10 @@ github.com/elastic/elastic-agent-autodiscover v0.9.2 h1:eBmru2v66HRRHOFf89rDl9OZ
 github.com/elastic/elastic-agent-autodiscover v0.9.2/go.mod h1:RNaHnOTYfNptSTQUyZYnjypxmrR5AaE6BIap/175F5c=
 github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7bMgXoT2DsHfolO2CHE=
 github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
-github.com/elastic/elastic-agent-libs v0.21.5 h1:YTMwaBPgOPvQnxBPv7fLpUqjw2HgY3ymC9TwdzdOD8U=
-github.com/elastic/elastic-agent-libs v0.21.5/go.mod h1:xSeIP3NtOIT4N2pPS4EyURmS1Q8mK0lWZ8Wd1Du6q3w=
-github.com/elastic/elastic-agent-system-metrics v0.11.11 h1:Qjh3Zef23PfGlG91AF+9ciNLNQf/8cDJ4CalnLZtV3g=
-github.com/elastic/elastic-agent-system-metrics v0.11.11/go.mod h1:GNqmKfvOt8PwORjbS6GllNdMfkLpOWyTa7P8oQq4E5o=
+github.com/elastic/elastic-agent-libs v0.23.0 h1:xpMKkrw59QUYWUx/q2TMpJU+6vwB3Mw1VPhRPcGhBMo=
+github.com/elastic/elastic-agent-libs v0.23.0/go.mod h1:xSeIP3NtOIT4N2pPS4EyURmS1Q8mK0lWZ8Wd1Du6q3w=
+github.com/elastic/elastic-agent-system-metrics v0.13.2 h1:R4ogKHESuWhWTtopnw/aHnBxxSZbxd7KHV4GefdwT2M=
+github.com/elastic/elastic-agent-system-metrics v0.13.2/go.mod h1:ezM1kzDUT+vWXFh5oK8QXB/AEB0UoLWqWA8rkRicFFo=
 github.com/elastic/elastic-transport-go/v8 v8.6.1 h1:h2jQRqH6eLGiBSN4eZbQnJLtL4bC5b4lfVFRjw2R4e4=
 github.com/elastic/elastic-transport-go/v8 v8.6.1/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=

libbeat/outputs/elasticsearch/client_test.go

@@ -93,7 +93,7 @@ func TestPublish(t *testing.T) {
 		reg := monitoring.NewRegistry()
 		client, err := NewClient(
 			clientSettings{
-				observer:      outputs.NewStats(reg),
+				observer:      outputs.NewStats(reg, logp.NewNopLogger()),
 				connection:    eslegclient.ConnectionSettings{URL: url},
 				indexSelector: testIndexSelector{},
 			},
@@ -106,7 +106,7 @@ func TestPublish(t *testing.T) {
 		reg := monitoring.NewRegistry()
 		client, err := NewClient(
 			clientSettings{
-				observer: outputs.NewStats(reg),
+				observer: outputs.NewStats(reg, logp.NewNopLogger()),
 				connection: eslegclient.ConnectionSettings{
 					URL:              url,
 					CompressionLevel: compressionLevel,

libbeat/outputs/kafka/client_test.go

@@ -51,7 +51,7 @@ func TestClientShutdownPanic(t *testing.T) {
 		beat.Info{
 			Beat:        "libbeat",
 			IndexPrefix: "testbeat"},
-		outputs.NewStats(monitoring.NewRegistry()), cfg)
+		outputs.NewStats(monitoring.NewRegistry(), logp.NewNopLogger()), cfg)
 	require.NoError(t, err, "could not create kafka output")
 
 	b := outest.NewBatch(

libbeat/outputs/kafka/config.go

@@ -313,6 +313,7 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err
 	k.MetricRegistry = adapter.GetGoMetrics(
 		monitoring.Default,
 		"libbeat.outputs",
+		log,
 		adapter.Rename("incoming-byte-rate", "read.bytes"),
 		adapter.Rename("outgoing-byte-rate", "write.bytes"),
 		adapter.Rename("request-latency-in-ms", "write.latency"),

libbeat/outputs/metrics.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/rcrowley/go-metrics"
 
+	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/monitoring"
 	"github.com/elastic/elastic-agent-libs/monitoring/adapter"
 )
@@ -83,7 +84,7 @@ type Stats struct {
 // NewStats creates a new Stats instance using a backing monitoring registry.
 // This function will create and register a number of metrics with the registry passed.
 // The registry must not be null.
-func NewStats(reg *monitoring.Registry) *Stats {
+func NewStats(reg *monitoring.Registry, logger *logp.Logger) *Stats {
 	obj := &Stats{
 		eventsBatches:    monitoring.NewUint(reg, "events.batches"),
 		eventsTotal:      monitoring.NewUint(reg, "events.total"),
@@ -105,7 +106,7 @@ func NewStats(reg *monitoring.Registry) *Stats {
 
 		sendLatencyMillis: metrics.NewUniformSample(1024),
 	}
-	_ = adapter.NewGoMetrics(reg, "write.latency", adapter.Accept).Register("histogram", metrics.NewHistogram(obj.sendLatencyMillis))
+	_ = adapter.NewGoMetrics(reg, "write.latency", logger, adapter.Accept).Register("histogram", metrics.NewHistogram(obj.sendLatencyMillis))
 	return obj
 }
 

libbeat/processors/dns/dns.go

@@ -64,7 +64,7 @@ func New(cfg *conf.C) (beat.Processor, error) {
 	)
 
 	log.Debugf("DNS processor config: %+v", c)
-	resolver, err := newMiekgResolver(metrics, c.Timeout, c.Transport, c.Nameservers...)
+	resolver, err := newMiekgResolver(metrics, c.Timeout, c.Transport, log, c.Nameservers...)
 	if err != nil {
 		return nil, err
 	}

libbeat/processors/dns/resolver.go

@@ -30,6 +30,7 @@ import (
 	"github.com/miekg/dns"
 	"github.com/rcrowley/go-metrics"
 
+	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/monitoring"
 	"github.com/elastic/elastic-agent-libs/monitoring/adapter"
 )
@@ -56,6 +57,7 @@ type miekgResolver struct {
 	registry     *monitoring.Registry
 	nsStatsMutex sync.RWMutex
 	nsStats      map[string]*nameserverStats
+	logger       *logp.Logger
 }
 
 type nameserverStats struct {
@@ -66,7 +68,7 @@ type nameserverStats struct {
 
 // newMiekgResolver returns a new miekgResolver. It returns an error if no
 // nameserver are given and none can be read from /etc/resolv.conf.
-func newMiekgResolver(reg *monitoring.Registry, timeout time.Duration, transport string, servers ...string) (*miekgResolver, error) {
+func newMiekgResolver(reg *monitoring.Registry, timeout time.Duration, transport string, logger *logp.Logger, servers ...string) (*miekgResolver, error) {
 	// Use /etc/resolv.conf if no nameservers are given. (Won't work for Windows).
 	if len(servers) == 0 {
 		config, err := dns.ClientConfigFromFile(etcResolvConf)
@@ -118,6 +120,7 @@ func newMiekgResolver(reg *monitoring.Registry, timeout time.Duration, transport
 		servers:  servers,
 		registry: reg,
 		nsStats:  map[string]*nameserverStats{},
+		logger:   logger,
 	}, nil
 }
 
@@ -249,7 +252,7 @@ func (res *miekgResolver) getOrCreateNameserverStats(ns string) *nameserverStats
 	}
 
 	//nolint:errcheck // Register should never fail because this is a new empty registry.
-	adapter.NewGoMetrics(reg, "request_duration", adapter.Accept).
+	adapter.NewGoMetrics(reg, "request_duration", res.logger, adapter.Accept).
 		Register("histogram", metrics.NewHistogram(stats.requestDuration))
 	res.nsStats[ns] = stats
 

libbeat/processors/dns/resolver_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/monitoring"
 )
 
@@ -38,7 +39,7 @@ func TestNewMiekgResolverWithIPv6(t *testing.T) {
 	const addr = `fe80::1%en0` // Example IPv6 address with zone index.
 
 	reg := monitoring.NewRegistry()
-	_, err := newMiekgResolver(reg.NewRegistry(logName), 0, "udp", addr)
+	_, err := newMiekgResolver(reg.NewRegistry(logName), 0, "udp", logp.NewNopLogger(), addr)
 	assert.NoError(t, err)
 }
 
@@ -52,7 +53,7 @@ func TestMiekgResolverLookupPTR(t *testing.T) {
 	}()
 
 	reg := monitoring.NewRegistry()
-	res, err := newMiekgResolver(reg.NewRegistry(logName), 0, "udp", addr)
+	res, err := newMiekgResolver(reg.NewRegistry(logName), 0, "udp", logp.NewNopLogger(), addr)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -103,7 +104,7 @@ func TestMiekgResolverLookupPTRTLS(t *testing.T) {
 
 	reg := monitoring.NewRegistry()
 
-	res, err := newMiekgResolver(reg.NewRegistry(logName), 0, "tls", addr)
+	res, err := newMiekgResolver(reg.NewRegistry(logName), 0, "tls", logp.NewNopLogger(), addr)
 	if err != nil {
 		t.Fatal(err)
 	}

libbeat/processors/script/javascript/javascript.go

@@ -33,6 +33,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/elastic-agent-libs/config"
+	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/monitoring"
 	"github.com/elastic/elastic-agent-libs/monitoring/adapter"
 	"github.com/elastic/elastic-agent-libs/paths"
@@ -96,7 +97,7 @@ func NewFromConfig(c Config, reg *monitoring.Registry) (beat.Processor, error) {
 		sessionPool: pool,
 		sourceProg:  prog,
 		sourceFile:  sourceFile,
-		stats:       getStats(c.Tag, reg),
+		stats:       getStats(c.Tag, reg, logp.NewLogger("")),
 	}, nil
 }
 
@@ -210,7 +211,7 @@ type processorStats struct {
 	processTime metrics.Sample
 }
 
-func getStats(id string, reg *monitoring.Registry) *processorStats {
+func getStats(id string, reg *monitoring.Registry, logger *logp.Logger) *processorStats {
 	if id == "" || reg == nil {
 		return nil
 	}
@@ -228,7 +229,7 @@ func getStats(id string, reg *monitoring.Registry) *processorStats {
 		exceptions:  monitoring.NewInt(processorReg, "exceptions"),
 		processTime: metrics.NewUniformSample(2048),
 	}
-	_ = adapter.NewGoMetrics(processorReg, "histogram", adapter.Accept).
+	_ = adapter.NewGoMetrics(processorReg, "histogram", logger, adapter.Accept).
 		Register("process_time", metrics.NewHistogram(stats.processTime))
 
 	return stats

libbeat/publisher/pipeline/controller_test.go

@@ -123,6 +123,7 @@ func TestSetEmptyOutputsSendsNilChannel(t *testing.T) {
 	// Just fill out enough to confirm what's sent to the event consumer,
 	// we don't want to start up real helper routines.
 	controller := outputController{
+		beat: beat.Info{},
 		consumer: &eventConsumer{
 			targetChan: make(chan consumerTarget, 2),
 		},

libbeat/publisher/pipeline/module.go

@@ -133,7 +133,7 @@ func loadOutput(
 		} else {
 			metrics = monitors.Metrics.NewRegistry("output")
 		}
-		outStats = outputs.NewStats(metrics)
+		outStats = outputs.NewStats(metrics, monitors.Logger)
 	}
 
 	outName, out, err := makeOutput(outStats)

metricbeat/module/linux/iostat/iostat.go

@@ -69,7 +69,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 // format. It publishes the event which is then forwarded to the output. In case
 // of an error set the Error field of mb.Event or simply call report.Error().
 func (m *MetricSet) Fetch(report mb.ReporterV2) error {
-	IOstats, err := diskio.IOCounters(m.includeDevices...)
+	IOstats, err := diskio.IOCounters(m.Logger(), m.includeDevices...)
 	if err != nil {
 		return fmt.Errorf("disk io counters: %w", err)
 	}

metricbeat/module/system/diskio/diskio.go

@@ -71,7 +71,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 // Fetch fetches disk IO metrics from the OS.
 func (m *MetricSet) Fetch(r mb.ReporterV2) error {
-	stats, err := diskio.IOCounters(m.includeDevices...)
+	stats, err := diskio.IOCounters(m.Logger(), m.includeDevices...)
 	if err != nil {
 		return fmt.Errorf("disk io counters: %w", err)
 	}

metricbeat/module/system/load/load.go

@@ -50,7 +50,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 // Fetch fetches system load metrics.
 func (m *MetricSet) Fetch(r mb.ReporterV2) error {
-	load, err := cpu.Load()
+	load, err := cpu.LoadWithLogger(m.Logger())
 	if err != nil {
 		return fmt.Errorf("failed to get CPU load values: %w", err)
 	}

metricbeat/module/system/process/process.go

@@ -93,6 +93,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 				RootfsMountpoint:  sys,
 				IgnoreRootCgroups: true,
 			},
+			Logger: base.Logger(),
 		},
 		perCPU:           config.IncludePerCPU,
 		degradeOnPartial: degradedConf.DegradeOnPartial,

packetbeat/protos/tcp/tcp.go

@@ -439,9 +439,11 @@ func newInputMetrics(id, device string, ports map[uint16]protos.Protocol) *input
 		arrivalPeriod:  metrics.NewUniformSample(1024),
 		processingTime: metrics.NewUniformSample(1024),
 	}
-	_ = adapter.NewGoMetrics(reg, "arrival_period", adapter.Accept).
+
+	//TODO: use local logger here
+	_ = adapter.NewGoMetrics(reg, "arrival_period", logp.NewLogger(""), adapter.Accept).
 		Register("histogram", metrics.NewHistogram(out.arrivalPeriod))
-	_ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept).
+	_ = adapter.NewGoMetrics(reg, "processing_time", logp.NewLogger(""), adapter.Accept).
 		Register("histogram", metrics.NewHistogram(out.processingTime))
 
 	out.device.Set(device)

packetbeat/protos/udp/udp.go

@@ -168,9 +168,11 @@ func newInputMetrics(id, device string, ports map[uint16]protos.Protocol) *input
 		arrivalPeriod:  metrics.NewUniformSample(1024),
 		processingTime: metrics.NewUniformSample(1024),
 	}
-	_ = adapter.NewGoMetrics(reg, "arrival_period", adapter.Accept).
+
+	// TODO: https://github.com/elastic/ingest-dev/issues/6000
+	_ = adapter.NewGoMetrics(reg, "arrival_period", logp.NewLogger(""), adapter.Accept).
 		Register("histogram", metrics.NewHistogram(out.arrivalPeriod))
-	_ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept).
+	_ = adapter.NewGoMetrics(reg, "processing_time", logp.NewLogger(""), adapter.Accept).
 		Register("histogram", metrics.NewHistogram(out.processingTime))
 
 	out.device.Set(device)