[Agent] Inject index with sane defaults to configuration

x-pack/agent/_meta/agent.docker.yml

@@ -8,12 +8,21 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: sample
+    use_output: default
+    inputs:
+      - type: logs
+        processors:
+        streams:
+          - dataset: sample.acccess
+            paths: /var/log/sample/access.log
+          - enabled: false
+            dataset: sample.error
+            paths: /var/log/sample/error.log
+      - type: metrics/system
+        streams:
+          - metricset: cpu
 
 management:
   # Mode of management, the agent support two modes of operation:

x-pack/agent/_meta/agent.yml

@@ -8,12 +8,21 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: sample
+    use_output: default
+    inputs:
+      - type: logs
+        processors:
+        streams:
+          - dataset: sample.acccess
+            paths: /var/log/sample/access.log
+          - enabled: false
+            dataset: sample.error
+            paths: /var/log/sample/error.log
+      - type: metrics/system
+        streams:
+          - metricset: cpu
 
 management:
   # Mode of management, the agent support two modes of operation:

x-pack/agent/_meta/common.p2.yml

@@ -8,12 +8,21 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: sample
+    use_output: default
+    inputs:
+      - type: logs
+        processors:
+        streams:
+          - dataset: sample.acccess
+            paths: /var/log/sample/access.log
+          - enabled: false
+            dataset: sample.error
+            paths: /var/log/sample/error.log
+      - type: metrics/system
+        streams:
+          - metricset: cpu
 
 management:
   # Mode of magement, the agent support two modes of operation:

x-pack/agent/_meta/common.reference.p2.yml

@@ -8,12 +8,21 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: sample
+    use_output: default
+    inputs:
+      - type: logs
+        processors:
+        streams:
+          - dataset: sample.acccess
+            paths: /var/log/sample/access.log
+          - enabled: false
+            dataset: sample.error
+            paths: /var/log/sample/error.log
+      - type: metrics/system
+        streams:
+          - metricset: cpu
 
 management:
   # Mode of management, the agent currently only support the following mode:

x-pack/agent/agent.docker.yml

@@ -8,12 +8,21 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: sample
+    use_output: default
+    inputs:
+      - type: logs
+        processors:
+        streams:
+          - dataset: sample.acccess
+            paths: /var/log/sample/access.log
+          - enabled: false
+            dataset: sample.error
+            paths: /var/log/sample/error.log
+      - type: metrics/system
+        streams:
+          - metricset: cpu
 
 management:
   # Mode of management, the agent support two modes of operation:

x-pack/agent/agent.reference.yml

@@ -13,12 +13,21 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: sample
+    use_output: default
+    inputs:
+      - type: logs
+        processors:
+        streams:
+          - dataset: sample.acccess
+            paths: /var/log/sample/access.log
+          - enabled: false
+            dataset: sample.error
+            paths: /var/log/sample/error.log
+      - type: metrics/system
+        streams:
+          - metricset: cpu
 
 management:
   # Mode of management, the agent currently only support the following mode:

x-pack/agent/agent.yml

@@ -13,12 +13,21 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: sample
+    use_output: default
+    inputs:
+      - type: logs
+        processors:
+        streams:
+          - dataset: sample.acccess
+            paths: /var/log/sample/access.log
+          - enabled: false
+            dataset: sample.error
+            paths: /var/log/sample/error.log
+      - type: metrics/system
+        streams:
+          - metricset: cpu
 
 management:
   # Mode of magement, the agent support two modes of operation:

x-pack/agent/pkg/agent/application/monitoring_decorator.go

@@ -5,19 +5,24 @@
 package application
 
 import (
+	"fmt"
+
 	"github.com/elastic/beats/x-pack/agent/pkg/agent/program"
 	"github.com/elastic/beats/x-pack/agent/pkg/agent/transpiler"
 )
 
 const (
-	monitoringName      = "FLEET_MONITORING"
-	programsKey         = "programs"
-	monitoringKey       = "monitoring"
-	monitoringOutputKey = "monitoring.elasticsearch"
-	enabledKey          = "monitoring.enabled"
-	outputKey           = "output"
-	outputsKey          = "outputs"
-	typeKey             = "type"
+	monitoringName            = "FLEET_MONITORING"
+	programsKey               = "programs"
+	monitoringKey             = "settings.monitoring"
+	monitoringUseOutputKey    = "settings.monitoring.use_output"
+	monitoringOutputFormatKey = "outputs.%s"
+	outputKey                 = "output"
+
+	enabledKey       = "settings.monitoring.enabled"
+	outputsKey       = "outputs"
+	elasticsearchKey = "elasticsearch"
+	typeKey          = "type"
 )
 
 func injectMonitoring(outputGroup string, rootAst *transpiler.AST, programsToRun []program.Program) ([]program.Program, error) {
@@ -35,8 +40,21 @@ func injectMonitoring(outputGroup string, rootAst *transpiler.AST, programsToRun
 		config = make(map[string]interface{})
 		config[enabledKey] = false
 	} else {
+		// get monitoring output name to be used
+		useOutputNode, found := transpiler.Lookup(rootAst, monitoringUseOutputKey)
+		if !found {
+			return programsToRun, nil
+		}
+
+		monitoringOutputNameKey, ok := useOutputNode.Value().(*transpiler.StrVal)
+		if !ok {
+			return programsToRun, nil
+		}
+
+		monitoringOutputName := monitoringOutputNameKey.String()
+
 		ast := rootAst.Clone()
-		if err := getMonitoringRule(outputGroup).Apply(ast); err != nil {
+		if err := getMonitoringRule(monitoringOutputName).Apply(ast); err != nil {
 			return programsToRun, err
 		}
 
@@ -63,8 +81,10 @@ func injectMonitoring(outputGroup string, rootAst *transpiler.AST, programsToRun
 }
 
 func getMonitoringRule(outputName string) *transpiler.RuleList {
+	monitoringOutputSelector := fmt.Sprintf(monitoringOutputFormatKey, outputName)
 	return transpiler.NewRuleList(
-		transpiler.Copy(monitoringOutputKey, outputKey),
+		transpiler.Copy(monitoringOutputSelector, outputKey),
+		transpiler.Rename(fmt.Sprintf("%s.%s", outputsKey, outputName), elasticsearchKey),
 		transpiler.Filter(monitoringKey, programsKey, outputKey),
 	)
 }

x-pack/agent/pkg/agent/application/monitoring_decorator_test.go

@@ -83,16 +83,11 @@ GROUPLOOP:
 }
 
 var inputConfigMap = map[string]interface{}{
-	"monitoring": map[string]interface{}{
-		"enabled": true,
-		"logs":    true,
-		"metrics": true,
-		"elasticsearch": map[string]interface{}{
-			"index_name": "general",
-			"pass":       "xxx",
-			"url":        "xxxxx",
-			"username":   "monitoring-uname",
-		},
+	"settings.monitoring": map[string]interface{}{
+		"enabled":    true,
+		"logs":       true,
+		"metrics":    true,
+		"use_output": "monitoring",
 	},
 	"outputs": map[string]interface{}{
 		"default": map[string]interface{}{
@@ -111,33 +106,39 @@ var inputConfigMap = map[string]interface{}{
 			"url":      "xxxxx",
 			"username": "xxx",
 		},
+		"monitoring": map[string]interface{}{
+			"type":       "elasticsearch",
+			"index_name": "general",
+			"pass":       "xxx",
+			"url":        "xxxxx",
+			"username":   "monitoring-uname",
+		},
 	},
-	"streams": []interface{}{
+	"datasources": []map[string]interface{}{
 		map[string]interface{}{
-			"type": "log",
-			"path": "/xxxx",
-			"processors": []interface{}{
+			"inputs": []map[string]interface{}{
 				map[string]interface{}{
-					"dissect": map[string]interface{}{
-						"tokenizer": "---",
+					"type": "log",
+					"streams": []map[string]interface{}{
+						map[string]interface{}{"paths": "/xxxx"},
+					},
+					"processors": []interface{}{
+						map[string]interface{}{
+							"dissect": map[string]interface{}{
+								"tokenizer": "---",
+							},
+						},
 					},
-				},
-			},
-			"output": map[string]interface{}{
-				"override": map[string]interface{}{
-					"index_name":      "my_service_logs",
-					"ingest_pipeline": "process_logs",
 				},
 			},
 		},
 		map[string]interface{}{
-			"type":     "metric/system",
-			"username": "xxxx",
-			"pass":     "yyy",
-			"output": map[string]interface{}{
-				"index_name": "mysql_metrics",
-				"use_output": "infosec1",
+			"inputs": []map[string]interface{}{
+				map[string]interface{}{
+					"type": "metrics/system",
+				},
 			},
+			"use_output": "infosec1",
 		},
 	},
 }

x-pack/agent/pkg/agent/operation/config/config.go

@@ -18,5 +18,5 @@ type Config struct {
 
 	DownloadConfig *artifact.Config `yaml:"download" config:"download"`
 
-	MonitoringConfig *monitoring.Config `yaml:"monitoring" config:"monitoring"`
+	MonitoringConfig *monitoring.Config `yaml:"settings.monitoring" config:"settings.monitoring"`
 }

x-pack/agent/pkg/agent/operation/monitoring.go

@@ -16,6 +16,7 @@ import (
 
 const (
 	monitoringName          = "FLEET_MONITORING"
+	settingsKey             = "settings"
 	monitoringKey           = "monitoring"
 	outputKey               = "output"
 	monitoringEnabledSubkey = "enabled"
@@ -97,15 +98,27 @@ func monitoringTags() map[app.Tag]string {
 }
 
 func isMonitoringEnabled(logger *logger.Logger, cfg map[string]interface{}) bool {
-	monitoringVal, found := cfg[monitoringKey]
+	settingsVal, found := cfg[settingsKey]
 	if !found {
-		logger.Error("operator.isMonitoringEnabled: monitoring not found in config")
+		logger.Error("operator.isMonitoringEnabled: settings not found in config")
+		return false
+	}
+
+	settingsMap, ok := settingsVal.(map[string]interface{})
+	if !ok {
+		logger.Error("operator.isMonitoringEnabled: settings not a map")
+		return false
+	}
+
+	monitoringVal, found := settingsMap[monitoringKey]
+	if !found {
+		logger.Error("operator.isMonitoringEnabled: settings.monitoring not found in config")
 		return false
 	}
 
 	monitoringMap, ok := monitoringVal.(map[string]interface{})
 	if !ok {
-		logger.Error("operator.isMonitoringEnabled: monitoring not a map")
+		logger.Error("operator.isMonitoringEnabled: settings.monitoring not a map")
 		return false
 	}