[Elastic Agent] Use Fleet agent ID as agent.id (#26394)

andrewkroh · mergify-bot · commit 43e9474ceeac · 2021-06-22T16:40:27.000Z
This updates the inject_agent_info rule to set the `agent.id` field to the value of the Fleet Agent ID. Previously this value was only added to the `elastic_agent.id` field, and the `agent.id` field was a random UUID generated the first time a Beat process was run. And each Beat process would have its own UUID. This change affects metricbeat, filebeat, heartbeat, osquerybeat, and packetbeat (these are the Beats that have an integration with Agent today). Heartbeat's Agent spec was missing the `inject_agent_info` so I added it. Closes #21121 (cherry picked from commit bb950bf)
diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc
@@ -113,3 +113,4 @@
 - Log output of container to $LOGS_PATH/elastic-agent-start.log when LOGS_PATH set {pull}25150[25150]
 - Use `filestream` input for internal log collection. {pull}25660[25660]
 - Enable agent to send custom headers to kibana/ES {pull}26275[26275]
+- Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue}21121[21121] {pull}26394[26394]
diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml
@@ -22,6 +22,10 @@ filebeat:
             id: agent-id
             version: 8.0.0
             snapshot: false
+      - add_fields:
+          target: "agent"
+          fields:
+            id: agent-id
 output:
   elasticsearch:
     enabled: true
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml
@@ -23,6 +23,10 @@ filebeat:
             id: agent-id
             version: 8.0.0
             snapshot: false
+      - add_fields:
+          target: "agent"
+          fields:
+            id: agent-id
 output:
   elasticsearch:
     hosts:
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml
@@ -24,6 +24,10 @@ filebeat:
             id: agent-id
             version: 8.0.0
             snapshot: false
+      - add_fields:
+          target: "agent"
+          fields:
+            id: agent-id
   - type: log
     paths:
       - /var/log/hello3.log
@@ -48,6 +52,10 @@ filebeat:
             id: agent-id
             version: 8.0.0
             snapshot: false
+      - add_fields:
+          target: "agent"
+          fields:
+            id: agent-id
 output:
   elasticsearch:
     hosts:
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-heartbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-heartbeat.yml
@@ -7,6 +7,17 @@ inputs:
   timeout: 16s
   wait: 1s
   data_stream.namespace: default
+  processors:
+    - add_fields:
+        target: 'elastic_agent'
+        fields:
+          id: agent-id
+          version: 8.0.0
+          snapshot: false
+    - add_fields:
+        target: 'agent'
+        fields:
+          id: agent-id
 output:
   elasticsearch:
     hosts:
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml
@@ -21,6 +21,10 @@ metricbeat:
           id: agent-id
           version: 8.0.0
           snapshot: false
+    - add_fields:
+        target: "agent"
+        fields:
+          id: agent-id
   - module: docker
     metricsets: [info]
     index: metrics-generic-default
@@ -42,6 +46,10 @@ metricbeat:
           id: agent-id
           version: 8.0.0
           snapshot: false
+    - add_fields:
+        target: "agent"
+        fields:
+          id: agent-id
   - module: apache
     metricsets: [info]
     index: metrics-generic-testing
@@ -66,6 +74,10 @@ metricbeat:
             id: agent-id
             version: 8.0.0
             snapshot: false
+      - add_fields:
+          target: "agent"
+          fields:
+            id: agent-id
 output:
   elasticsearch:
     hosts: [127.0.0.1:9200, 127.0.0.1:9300]
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-packetbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-packetbeat.yml
@@ -7,6 +7,10 @@ inputs:
           id: agent-id
           version: 8.0.0
           snapshot: false
+    - add_fields:
+        target: 'agent'
+        fields:
+          id: agent-id
   streams:
   - type: flow
     timeout: 10s
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/synthetics_config-heartbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/synthetics_config-heartbeat.yml
@@ -7,6 +7,17 @@ inputs:
   timeout: 16s
   wait: 1s
   data_stream.namespace: default
+  processors:
+    - add_fields:
+        target: 'elastic_agent'
+        fields:
+          id: agent-id
+          version: 8.0.0
+          snapshot: false
+    - add_fields:
+        target: 'agent'
+        fields:
+          id: agent-id
 - type: synthetics/tcp
   id: unique-tcp-id
   name: my-tcp
@@ -15,6 +26,17 @@ inputs:
   timeout: 16s
   wait: 1s
   data_stream.namespace: default
+  processors:
+    - add_fields:
+        target: 'elastic_agent'
+        fields:
+          id: agent-id
+          version: 8.0.0
+          snapshot: false
+    - add_fields:
+        target: 'agent'
+        fields:
+          id: agent-id
 - type: synthetics/icmp
   id: unique-icmp-id
   name: my-icmp
@@ -26,6 +48,17 @@ inputs:
   timeout: 16s
   wait: 1s
   data_stream.namespace: default
+  processors:
+    - add_fields:
+        target: 'elastic_agent'
+        fields:
+          id: agent-id
+          version: 8.0.0
+          snapshot: false
+    - add_fields:
+        target: 'agent'
+        fields:
+          id: agent-id
 output:
   elasticsearch:
     hosts: [127.0.0.1:9200, 127.0.0.1:9300]
diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go
@@ -775,7 +775,7 @@ func (r *InjectAgentInfoRule) Apply(agentInfo AgentInfo, ast *AST) (err error) {
 			return errors.New("InjectAgentInfoRule: processors is not a list")
 		}
 
-		// elastic.agent
+		// elastic_agent
 		processorMap := &Dict{value: make([]Node, 0)}
 		processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "elastic_agent"}})
 		processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{
@@ -785,6 +785,15 @@ func (r *InjectAgentInfoRule) Apply(agentInfo AgentInfo, ast *AST) (err error) {
 		}}})
 		addFieldsMap := &Dict{value: []Node{&Key{"add_fields", processorMap}}}
 		processorsList.value = mergeStrategy("").InjectItem(processorsList.value, addFieldsMap)
+
+		// agent.id
+		processorMap = &Dict{value: make([]Node, 0)}
+		processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "agent"}})
+		processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{
+			&Key{name: "id", value: &StrVal{value: agentInfo.AgentID()}},
+		}}})
+		addFieldsMap = &Dict{value: []Node{&Key{"add_fields", processorMap}}}
+		processorsList.value = mergeStrategy("").InjectItem(processorsList.value, addFieldsMap)
 	}
 
 	return nil
diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules_test.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules_test.go
@@ -189,6 +189,10 @@ inputs:
             id: agent-id
             snapshot: false
             version: 8.0.0
+      - add_fields:
+          target: agent
+          fields:
+            id: agent-id
   - name: With processors
     type: file
     processors:
@@ -202,6 +206,10 @@ inputs:
             id: agent-id
             snapshot: false
             version: 8.0.0
+      - add_fields:
+          target: agent
+          fields:
+            id: agent-id
 `,
 			rule: &RuleList{
 				Rules: []Rule{
diff --git a/x-pack/elastic-agent/spec/heartbeat.yml b/x-pack/elastic-agent/spec/heartbeat.yml
@@ -14,6 +14,7 @@ rules:
       key: enabled
       values:
         - true
+  - inject_agent_info: {}
   - filter:
       selectors:
         - inputs
