[9.1] (backport #11998) (bugfix) log level does not change when standalone agent is reloaded or when otel runtime is used

changelog/fragments/1766556100-reload-log-level.yaml

@@ -0,0 +1,45 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: Fix reloading agent.logging.level for standalone Elastic Agent 
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+# pr: https://github.com/owner/repo/1234
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

internal/pkg/agent/application/application.go

@@ -240,7 +240,22 @@
 		return nil, nil, nil, errors.New(err, "failed to initialize composable controller")
 	}
 
+<<<<<<< HEAD
 	otelManager := otelmanager.NewOTelManager(log.Named("otel_manager"), baseLogger)
+=======
+	otelManager, err := otelmanager.NewOTelManager(
+		log.Named("otel_manager"),
+		logLevel,
+		baseLogger,
+		agentInfo,
+		cfg.Settings.Collector,
+		monitor.ComponentMonitoringConfig,
+		otelmanager.CollectorStopTimeout,
+	)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to create otel manager: %w", err)
+	}
+>>>>>>> 85b7e9932 ((bugfix) log level does not change when standalone agent is reloaded or when otel runtime is used (#11998))
 	coord := coordinator.New(log, cfg, logLevel, agentInfo, specs, reexec, upgrader, runtime, configMgr, varsManager, caps, monitor, isManaged, otelManager, actionAcker, initialUpgradeDetails, compModifiers...)
 	if managed != nil {
 		// the coordinator requires the config manager as well as in managed-mode the config manager requires the

internal/pkg/agent/application/coordinator/coordinator.go

@@ -139,8 +139,13 @@
 type OTelManager interface {
 	Runner
 
+<<<<<<< HEAD
 	// Update updates the current configuration for OTel.
 	Update(cfg *confmap.Conf)
+=======
+	// Update updates the current plain configuration for the otel collector and components.
+	Update(*confmap.Conf, *monitoringCfg.MonitoringConfig, logp.Level, []component.Component)
+>>>>>>> 85b7e9932 ((bugfix) log level does not change when standalone agent is reloaded or when otel runtime is used (#11998))
 
 	// Watch returns the chanel to watch for configuration changes.
 	Watch() <-chan *status.AggregateStatus
@@ -1458,10 +1463,20 @@
 
 	// c.setProtection(protectionConfig)
 
-	if c.vars != nil {
-		return c.refreshComponentModel(ctx)
+	// check if log level has changed for standalone elastic-agent
+	// we'd have to update both the periodic and once config watchers and refactor initialization in application.go to do otherwise.
+	if c.agentInfo.IsStandalone() {
+		ll := currentCfg.Settings.LoggingConfig.Level
+		if ll != c.state.LogLevel {
+			// set log level for the coordinator
+			c.setLogLevel(ll)
+			// set global log level
+			logger.SetLevel(ll)
+			c.logger.Infof("log level changed to %s", ll.String())
+		}
 	}
-	return nil
+
+	return c.refreshComponentModel(ctx)
 }
 
 // Generate the AST for a new incoming configuration and, if successful,
@@ -1574,10 +1589,13 @@
 
 // Called on the main Coordinator goroutine.
 func (c *Coordinator) processLogLevel(ctx context.Context, ll logp.Level) {
-	c.setLogLevel(ll)
-	err := c.refreshComponentModel(ctx)
-	if err != nil {
-		c.logger.Errorf("updating log level: %s", err.Error())
+	// do not refresh component model if log level did not change
+	if c.state.LogLevel != ll {
+		c.setLogLevel(ll)
+		err := c.refreshComponentModel(ctx)
+		if err != nil {
+			c.logger.Errorf("updating log level: %s", err.Error())
+		}
 	}
 }
 
@@ -1650,6 +1668,7 @@
 		}
 		c.logger.With("component_ids", componentIDs).Warn("The Otel runtime manager is HIGHLY EXPERIMENTAL and only intended for testing. Use at your own risk.")
 	}
+<<<<<<< HEAD
 	if componentOtelCfg != nil {
 		err := finalOtelCfg.Merge(componentOtelCfg)
 		if err != nil {
@@ -1672,6 +1691,9 @@
 	c.otelMgr.Update(finalOtelCfg)
 	c.finalOtelCfg = finalOtelCfg
 	return nil
+=======
+	c.otelMgr.Update(c.otelCfg, c.currentCfg.Settings.MonitoringConfig, c.state.LogLevel, otelModel.Components)
+>>>>>>> 85b7e9932 ((bugfix) log level does not change when standalone agent is reloaded or when otel runtime is used (#11998))
 }
 
 // splitModelBetweenManager splits the model components between the runtime manager and the otel manager.

internal/pkg/agent/application/coordinator/coordinator_test.go

@@ -1396,6 +1396,44 @@
 }
 
 func (f *fakeOTelManager) Errors() <-chan error {
+<<<<<<< HEAD
+=======
+	return f.errChan
+}
+
+func (f *fakeOTelManager) Update(cfg *confmap.Conf, monitoring *monitoringCfg.MonitoringConfig, ll logp.Level, components []component.Component) {
+	var collectorResult, componentResult error
+	if f.updateCollectorCallback != nil {
+		collectorResult = f.updateCollectorCallback(cfg)
+	}
+	if f.errChan != nil && collectorResult != nil {
+		// If a reporting channel is set, send the collectorResult to it
+		f.errChan <- collectorResult
+	}
+	if f.updateComponentCallback != nil {
+		componentResult = f.updateComponentCallback(components)
+	}
+	if f.errChan != nil && componentResult != nil {
+		// If a reporting channel is set, send the componentResult to it
+		f.errChan <- componentResult
+	}
+}
+
+func (f *fakeOTelManager) WatchCollector() <-chan *status.AggregateStatus {
+	return f.collectorStatusChan
+}
+
+func (f *fakeOTelManager) WatchComponents() <-chan []runtime.ComponentComponentState {
+	return f.componentStateChan
+}
+
+func (f *fakeOTelManager) MergedOtelConfig() *confmap.Conf { return nil }
+
+func (f *fakeOTelManager) PerformDiagnostics(ctx context.Context, reqs ...runtime.ComponentUnitDiagnosticRequest) []runtime.ComponentUnitDiagnostic {
+	if f.performDiagnosticsCallback != nil {
+		return f.performDiagnosticsCallback(ctx, reqs...)
+	}
+>>>>>>> 85b7e9932 ((bugfix) log level does not change when standalone agent is reloaded or when otel runtime is used (#11998))
 	return nil
 }
 

internal/pkg/agent/application/coordinator/coordinator_unit_test.go

@@ -452,11 +452,18 @@
 		}
 	}()
 
+<<<<<<< HEAD
 	upgradeMgr, err := upgrade.NewUpgrader(
 		log,
 		&artifact.Config{},
 		&info.AgentInfo{},
 	)
+=======
+	tmpDir := t.TempDir()
+	agentInfo, err := info.NewAgentInfo(ctx, false)
+	require.NoError(t, err)
+	upgradeMgr, err := upgrade.NewUpgrader(log, &artifact.Config{}, nil, agentInfo, new(upgrade.AgentWatcherHelper), ttl.NewTTLMarkerRegistry(nil, tmpDir))
+>>>>>>> 85b7e9932 ((bugfix) log level does not change when standalone agent is reloaded or when otel runtime is used (#11998))
 	require.NoError(t, err, "errored when creating a new upgrader")
 
 	// Channels have buffer length 1, so we don't have to run on multiple
@@ -488,6 +495,7 @@
 		ast:                emptyAST(t),
 		componentPIDTicker: time.NewTicker(time.Second * 30),
 		secretMarkerFunc:   testSecretMarkerFunc,
+		agentInfo:          agentInfo,
 	}
 
 	// Send an invalid config update and confirm that Coordinator reports
@@ -580,6 +588,9 @@
 	defer cancel()
 	logger := logp.NewLogger("testing")
 
+	agentInfo, err := info.NewAgentInfo(t.Context(), false)
+	require.NoError(t, err)
+
 	// Channels have buffer length 1 so we don't have to run on multiple
 	// goroutines.
 	stateChan := make(chan State, 1)
@@ -605,6 +616,7 @@
 		ast:                emptyAST(t),
 		componentPIDTicker: time.NewTicker(time.Second * 30),
 		secretMarkerFunc:   testSecretMarkerFunc,
+		agentInfo:          agentInfo,
 	}
 
 	// This configuration produces a valid AST but its EQL condition is

internal/pkg/agent/application/periodic.go

@@ -17,6 +17,7 @@ import (
 	"github.com/elastic/elastic-agent/pkg/core/logger"
 )
 
+// periodic checks for local configuration changes
 type periodic struct {
 	log      *logger.Logger
 	period   time.Duration
@@ -147,6 +148,7 @@ func newPeriodic(
 	}
 }
 
+// localConfigChange implements coordinator.ConfigChange for local file changes.
 type localConfigChange struct {
 	cfg *config.Config
 }

internal/pkg/agent/cmd/run.go

@@ -302,19 +302,17 @@ func runElasticAgent(
 			errors.M(errors.MetaKeyPath, pathConfigFile))
 	}
 
+	// Set the initial log level (either default or from config file)
+	logger.SetLevel(logLvl)
+
 	// Ensure that the log level now matches what is configured in the agentInfo.
-	if agentInfo.LogLevel() != "" {
-		var lvl logp.Level
-		err = lvl.Unpack(agentInfo.LogLevel())
-		if err != nil {
-			l.Error(errors.New(err, "failed to parse agent information log level"))
-		} else {
-			logLvl = lvl
-			logger.SetLevel(lvl)
-		}
+	var lvl logp.Level
+	err = lvl.Unpack(agentInfo.LogLevel())
+	if err != nil {
+		l.Error(errors.New(err, "failed to parse agent information log level"))
 	} else {
-		// Set the initial log level (either default or from config file)
-		logger.SetLevel(logLvl)
+		logLvl = lvl
+		logger.SetLevel(lvl)
 	}
 
 	// initiate agent watcher

internal/pkg/otel/manager/execution.go

@@ -0,0 +1,36 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package manager
+
+import (
+	"context"
+	"time"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/status"
+	"go.opentelemetry.io/collector/confmap"
+
+	"github.com/elastic/elastic-agent/pkg/core/logger"
+)
+
+type collectorExecution interface {
+	// startCollector starts the otel collector with the given arguments, returning a handle allowing it to be stopped.
+	// Cancelling the context will stop all goroutines involved in the execution.
+	// The collector will report status events in the statusCh channel and errors on errCh in a non-blocking fashion,
+	// draining the channel before writing to it.
+	// After the collector exits, it will emit an error describing the exit status (nil if successful) and a nil status.
+	// Parameters:
+	//   - cfg: Configuration for the collector.
+	//   - errCh: Process exit errors are sent to the errCh channel
+	//   - statusCh: Collector's status updates are sent to statusCh channel.
+	//   - forceFetchStatusCh: Channel that is used to trigger a forced status update.
+	startCollector(ctx context.Context, logLevel string, baseLogger *logger.Logger, logger *logger.Logger, cfg *confmap.Conf, errCh chan error, statusCh chan *status.AggregateStatus, forceFetchStatusCh chan struct{}) (collectorHandle, error)
+}
+
+type collectorHandle interface {
+	// Stop stops and waits for collector to exit gracefully within the given duration. Note that if the collector
+	// doesn't exit within that time, it will be killed and then it will wait an extra second for it to ensure it's
+	// really stopped.
+	Stop(waitTime time.Duration)
+}