[Fleet] Enable log introspections

x-pack/agent/_meta/agent.docker.yml

@@ -108,3 +108,11 @@ retry:
   # With 30s delay and 3 retries: 30, 60, 120s
   # Default is false
   exponential: false
+
+monitoring:
+  # enabled turns on monitoring of running processes
+  enabled: false
+  # enables log monitoring
+  logs: false
+  # enables metrics monitoring
+  metrics: false

x-pack/agent/_meta/agent.yml

@@ -108,3 +108,11 @@ retry:
   # With 30s delay and 3 retries: 30, 60, 120s
   # Default is false
   exponential: false
+
+monitoring:
+  # enabled turns on monitoring of running processes
+  enabled: false
+  # enables log monitoring
+  logs: false
+  # enables metrics monitoring
+  metrics: false

x-pack/agent/_meta/common.p2.yml

@@ -104,3 +104,11 @@ retry:
   # With 30s delay and 3 retries: 30, 60, 120s
   # Default is false
   exponential: false
+
+monitoring:
+  # enabled turns on monitoring of running processes
+  enabled: false
+  # enables log monitoring
+  logs: false
+  # enables metrics monitoring
+  metrics: false

x-pack/agent/_meta/common.reference.p2.yml

@@ -107,3 +107,11 @@ retry:
   # With 30s delay and 3 retries: 30, 60, 120s
   # Default is false
   exponential: false
+
+monitoring:
+  # enabled turns on monitoring of running processes
+  enabled: false
+  # enables log monitoring
+  logs: false
+  # enables metrics monitoring
+  metrics: false

x-pack/agent/agent.docker.yml

@@ -108,3 +108,11 @@ retry:
   # With 30s delay and 3 retries: 30, 60, 120s
   # Default is false
   exponential: false
+
+monitoring:
+  # enabled turns on monitoring of running processes
+  enabled: false
+  # enables log monitoring
+  logs: false
+  # enables metrics monitoring
+  metrics: false

x-pack/agent/agent.reference.yml

@@ -112,3 +112,11 @@ retry:
   # With 30s delay and 3 retries: 30, 60, 120s
   # Default is false
   exponential: false
+
+monitoring:
+  # enabled turns on monitoring of running processes
+  enabled: false
+  # enables log monitoring
+  logs: false
+  # enables metrics monitoring
+  metrics: false

x-pack/agent/agent.yml

@@ -109,3 +109,11 @@ retry:
   # With 30s delay and 3 retries: 30, 60, 120s
   # Default is false
   exponential: false
+
+monitoring:
+  # enabled turns on monitoring of running processes
+  enabled: false
+  # enables log monitoring
+  logs: false
+  # enables metrics monitoring
+  metrics: false

x-pack/agent/pkg/agent/application/emitter.go

@@ -15,7 +15,9 @@ import (
 	"github.com/elastic/beats/x-pack/agent/pkg/core/logger"
 )
 
-func emitter(log *logger.Logger, router *router) emitterFunc {
+type decoratorFunc = func(*transpiler.AST, []program.Program) ([]program.Program, error)
+
+func emitter(log *logger.Logger, router *router, decorators ...decoratorFunc) emitterFunc {
 	return func(files []string) error {
 		c, err := config.LoadFiles(files...)
 		if err != nil {
@@ -41,6 +43,13 @@ func emitter(log *logger.Logger, router *router) emitterFunc {
 			return err
 		}
 
+		for _, decorator := range decorators {
+			programsToRun, err = decorator(ast, programsToRun)
+			if err != nil {
+				return err
+			}
+		}
+
 		grouped := map[routingKey][]program.Program{
 			defautlRK: programsToRun,
 		}

x-pack/agent/pkg/agent/application/local_mode.go

@@ -58,7 +58,7 @@ func newLocal(
 	}
 
 	discover := discoverer(pathConfigFile, c.Path)
-	emit := emitter(log, router)
+	emit := emitter(log, router, injectMonitoring)
 
 	var cfgSource source
 	if !c.Reload.Enabled {

x-pack/agent/pkg/agent/application/monitoring_decorator.go

@@ -0,0 +1,66 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package application
+
+import (
+	"github.com/elastic/beats/x-pack/agent/pkg/agent/program"
+	"github.com/elastic/beats/x-pack/agent/pkg/agent/transpiler"
+)
+
+const (
+	monitoringName = "FLEET_MONITORING"
+	programsKey    = "programs"
+	monitoringKey  = "monitoring"
+	enabledKey     = "monitoring.enabled"
+	outputKey      = "output"
+)
+
+func injectMonitoring(rootAst *transpiler.AST, programsToRun []program.Program) ([]program.Program, error) {
+	var err error
+	monitoringProgram := program.Program{
+		Spec: program.Spec{
+			Name: monitoringName,
+			Cmd:  monitoringName,
+		},
+	}
+
+	var config map[string]interface{}
+
+	if _, found := transpiler.Lookup(rootAst, monitoringKey); !found {
+		config = make(map[string]interface{})
+		config[enabledKey] = false
+	} else {
+		ast := rootAst.Clone()
+		if err := getMonitoringRule().Apply(ast); err != nil {
+			return programsToRun, err
+		}
+
+		config, err = ast.Map()
+		if err != nil {
+			return programsToRun, err
+		}
+
+		programList := make([]string, 0, len(programsToRun))
+		for _, p := range programsToRun {
+			programList = append(programList, p.Spec.Cmd)
+		}
+		// making program list part of the config
+		// so it will get regenerated with every change
+		config[programsKey] = programList
+	}
+
+	monitoringProgram.Config, err = transpiler.NewAST(config)
+	if err != nil {
+		return programsToRun, err
+	}
+
+	return append(programsToRun, monitoringProgram), nil
+}
+
+func getMonitoringRule() *transpiler.RuleList {
+	return transpiler.NewRuleList(
+		transpiler.Filter(monitoringKey, programsKey, outputKey),
+	)
+}

x-pack/agent/pkg/agent/application/stream.go

@@ -56,7 +56,7 @@ func (b *operatorStream) Execute(cfg *configRequest) error {
 func streamFactory(cfg *config.Config, client sender, r reporter) func(*logger.Logger, routingKey) (stream, error) {
 	return func(log *logger.Logger, id routingKey) (stream, error) {
 		// new operator per stream to isolate processes without using tags
-		operator, err := newOperator(log, cfg, r)
+		operator, err := newOperator(log, id, cfg, r)
 		if err != nil {
 			return nil, err
 		}
@@ -68,7 +68,7 @@ func streamFactory(cfg *config.Config, client sender, r reporter) func(*logger.L
 	}
 }
 
-func newOperator(log *logger.Logger, config *config.Config, r reporter) (*operation.Operator, error) {
+func newOperator(log *logger.Logger, id routingKey, config *config.Config, r reporter) (*operation.Operator, error) {
 	operatorConfig := &operatorCfg.Config{}
 	if err := config.Unpack(&operatorConfig); err != nil {
 		return nil, err
@@ -87,6 +87,7 @@ func newOperator(log *logger.Logger, config *config.Config, r reporter) (*operat
 
 	return operation.NewOperator(
 		log,
+		id,
 		config,
 		fetcher,
 		installer,

x-pack/agent/pkg/agent/errors/types.go

@@ -42,4 +42,7 @@ var readableTypes = map[ErrorType]string{
 	TypeConfig:           "CONFIG",
 	TypePath:             "PATH",
 	TypeApplicationCrash: "CRASH",
+	TypeNetwork:          "NETWORK",
+	TypeFilesystem:       "FILESYSTEM",
+	TypeSecurity:         "SECURITY",
 }

x-pack/agent/pkg/agent/operation/config/config.go

@@ -6,6 +6,7 @@ package config
 
 import (
 	"github.com/elastic/beats/x-pack/agent/pkg/artifact"
+	"github.com/elastic/beats/x-pack/agent/pkg/core/plugin/app/monitoring"
 	"github.com/elastic/beats/x-pack/agent/pkg/core/plugin/process"
 	"github.com/elastic/beats/x-pack/agent/pkg/core/plugin/retry"
 )
@@ -16,4 +17,6 @@ type Config struct {
 	RetryConfig   *retry.Config   `yaml:"retry" config:"retry"`
 
 	DownloadConfig *artifact.Config `yaml:"download" config:"download"`
+
+	MonitoringConfig *monitoring.Config `yaml:"monitoring" config:"monitoring"`
 }