Skip to content

Commit ce14415

Browse files
michalpristasmichalpristas
committed
[Elastic Agent] Enable configuring monitoring namespace (#26439)
[Elastic Agent] Enable configuring monitoring namespace (#26439)
1 parent 9c89a29 commit ce14415

File tree

16 files changed

+594
-16
lines changed

16 files changed

+594
-16
lines changed

‎x-pack/elastic-agent/CHANGELOG.next.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,4 +116,5 @@
116116
- Use `filestream` input for internal log collection. {pull}25660[25660]
117117
- Enable agent to send custom headers to kibana/ES {pull}26275[26275]
118118
- Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue}21121[21121] {pull}26394[26394]
119+
- Enable configuring monitoring namespace {issue}26439[26439]
119120
- Communicate with Fleet Server over HTTP2. {pull}26474[26474]

‎x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ func InjectMonitoring(agentInfo *info.AgentInfo, outputGroup string, rootAst *tr
4949
transpiler.NewKey("logs", transpiler.NewBoolVal(true)),
5050
transpiler.NewKey("metrics", transpiler.NewBoolVal(true)),
5151
transpiler.NewKey("use_output", transpiler.NewStrVal("default")),
52+
transpiler.NewKey("namespace", transpiler.NewStrVal("default")),
5253
})
5354

5455
transpiler.Insert(rootAst, transpiler.NewKey("monitoring", monitoringNode), "settings")

‎x-pack/elastic-agent/pkg/agent/operation/monitoring.go‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -161,11 +161,12 @@ func (o *Operator) generateMonitoringSteps(version, outputType string, output in
161161
var steps []configrequest.Step
162162
watchLogs := o.monitor.WatchLogs()
163163
watchMetrics := o.monitor.WatchMetrics()
164+
monitoringNamespace := o.monitor.MonitoringNamespace()
164165

165166
// generate only when monitoring is running (for config refresh) or
166167
// state changes (turning on/off)
167168
if watchLogs != o.isMonitoringLogs() || watchLogs {
168-
fbConfig, any := o.getMonitoringFilebeatConfig(outputType, output)
169+
fbConfig, any := o.getMonitoringFilebeatConfig(outputType, output, monitoringNamespace)
169170
stepID := configrequest.StepRun
170171
if !watchLogs || !any {
171172
stepID = configrequest.StepRemove
@@ -182,7 +183,7 @@ func (o *Operator) generateMonitoringSteps(version, outputType string, output in
182183
steps = append(steps, filebeatStep)
183184
}
184185
if watchMetrics != o.isMonitoringMetrics() || watchMetrics {
185-
mbConfig, any := o.getMonitoringMetricbeatConfig(outputType, output)
186+
mbConfig, any := o.getMonitoringMetricbeatConfig(outputType, output, monitoringNamespace)
186187
stepID := configrequest.StepRun
187188
if !watchMetrics || !any {
188189
stepID = configrequest.StepRemove
@@ -215,12 +216,12 @@ func loadSpecFromSupported(processName string) program.Spec {
215216
}
216217
}
217218

218-
func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interface{}) (map[string]interface{}, bool) {
219+
func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interface{}, monitoringNamespace string) (map[string]interface{}, bool) {
219220
inputs := []interface{}{
220221
map[string]interface{}{
221222
"type": "filestream",
222223
"parsers": []map[string]interface{}{
223-
map[string]interface{}{
224+
{
224225
"ndjson": map[string]interface{}{
225226
"overwrite_keys": true,
226227
"message_key": "message",
@@ -233,15 +234,15 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa
233234
filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log"),
234235
filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log*"),
235236
},
236-
"index": "logs-elastic_agent-default",
237+
"index": fmt.Sprintf("logs-elastic_agent-%s", monitoringNamespace),
237238
"processors": []map[string]interface{}{
238239
{
239240
"add_fields": map[string]interface{}{
240241
"target": "data_stream",
241242
"fields": map[string]interface{}{
242243
"type": "logs",
243244
"dataset": "elastic_agent",
244-
"namespace": "default",
245+
"namespace": monitoringNamespace,
245246
},
246247
},
247248
},
@@ -280,23 +281,23 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa
280281
inputs = append(inputs, map[string]interface{}{
281282
"type": "filestream",
282283
"parsers": []map[string]interface{}{
283-
map[string]interface{}{
284+
{
284285
"ndjson": map[string]interface{}{
285286
"overwrite_keys": true,
286287
"message_key": "message",
287288
},
288289
},
289290
},
290291
"paths": paths,
291-
"index": fmt.Sprintf("logs-elastic_agent.%s-default", name),
292+
"index": fmt.Sprintf("logs-elastic_agent.%s-%s", name, monitoringNamespace),
292293
"processors": []map[string]interface{}{
293294
{
294295
"add_fields": map[string]interface{}{
295296
"target": "data_stream",
296297
"fields": map[string]interface{}{
297298
"type": "logs",
298299
"dataset": fmt.Sprintf("elastic_agent.%s", name),
299-
"namespace": "default",
300+
"namespace": monitoringNamespace,
300301
},
301302
},
302303
},
@@ -345,7 +346,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa
345346
return result, true
346347
}
347348

348-
func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output interface{}) (map[string]interface{}, bool) {
349+
func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output interface{}, monitoringNamespace string) (map[string]interface{}, bool) {
349350
hosts := o.getMetricbeatEndpoints()
350351
if len(hosts) == 0 {
351352
return nil, false
@@ -359,15 +360,15 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter
359360
"metricsets": []string{"stats", "state"},
360361
"period": "10s",
361362
"hosts": endpoints,
362-
"index": fmt.Sprintf("metrics-elastic_agent.%s-default", name),
363+
"index": fmt.Sprintf("metrics-elastic_agent.%s-%s", name, monitoringNamespace),
363364
"processors": []map[string]interface{}{
364365
{
365366
"add_fields": map[string]interface{}{
366367
"target": "data_stream",
367368
"fields": map[string]interface{}{
368369
"type": "metrics",
369370
"dataset": fmt.Sprintf("elastic_agent.%s", name),
370-
"namespace": "default",
371+
"namespace": monitoringNamespace,
371372
},
372373
},
373374
},
@@ -397,15 +398,15 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter
397398
"period": "10s",
398399
"path": "/stats",
399400
"hosts": endpoints,
400-
"index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName),
401+
"index": fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace),
401402
"processors": []map[string]interface{}{
402403
{
403404
"add_fields": map[string]interface{}{
404405
"target": "data_stream",
405406
"fields": map[string]interface{}{
406407
"type": "metrics",
407408
"dataset": fmt.Sprintf("elastic_agent.%s", fixedAgentName),
408-
"namespace": "default",
409+
"namespace": monitoringNamespace,
409410
},
410411
},
411412
},
@@ -480,15 +481,15 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter
480481
"period": "10s",
481482
"path": "/stats",
482483
"hosts": []string{beats.AgentPrefixedMonitoringEndpoint(o.config.DownloadConfig.OS(), o.config.MonitoringConfig.HTTP)},
483-
"index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName),
484+
"index": fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace),
484485
"processors": []map[string]interface{}{
485486
{
486487
"add_fields": map[string]interface{}{
487488
"target": "data_stream",
488489
"fields": map[string]interface{}{
489490
"type": "metrics",
490491
"dataset": fmt.Sprintf("elastic_agent.%s", fixedAgentName),
491-
"namespace": "default",
492+
"namespace": monitoringNamespace,
492493
},
493494
},
494495
},

‎x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,9 @@ func (b *testMonitor) Reload(cfg *config.Config) error { return nil }
215215
// IsMonitoringEnabled returns true if monitoring is configured.
216216
func (b *testMonitor) IsMonitoringEnabled() bool { return b.monitorLogs || b.monitorMetrics }
217217

218+
// MonitoringNamespace returns monitoring namespace configured.
219+
func (b *testMonitor) MonitoringNamespace() string { return "default" }
220+
218221
// WatchLogs return true if monitoring is configured and monitoring logs is enabled.
219222
func (b *testMonitor) WatchLogs() bool { return b.monitorLogs }
220223

‎x-pack/elastic-agent/pkg/agent/program/program_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,10 @@ func TestConfiguration(t *testing.T) {
383383
empty bool
384384
err bool
385385
}{
386+
"namespace": {
387+
programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"},
388+
expected: 6,
389+
},
386390
"single_config": {
387391
programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"},
388392
expected: 6,

‎x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
fleet:
2+
enabled: true
3+
access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw
4+
protocol: https
5+
hosts: [ localhost:5601 ]
6+
timeout: 30s
7+
agent:
8+
id: fleet-agent-id
9+
logging.level: error
10+
host:
11+
id: host-agent-id
12+
13+
output:
14+
elasticsearch:
15+
hosts:
16+
- "127.0.0.1:9200"
17+
- "127.0.0.1:9300"
18+
namespace: test_namespace
19+
username: elastic
20+
password: changeme
21+
api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA
22+
ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y=
23+
24+
inputs:
25+
- id: endpoint-id
26+
type: endpoint
27+
name: endpoint-1
28+
enabled: true
29+
package:
30+
name: endpoint
31+
version: 0.3.0
32+
data_stream:
33+
namespace: default
34+
artifact_manifest:
35+
schema_version: v22
36+
manifest_version: v21
37+
artifacts:
38+
- endpoint-allowlist-windows:
39+
sha256: 1234
40+
size: 2
41+
url: /relative/path/to/endpoint-allowlist-windows
42+
- endpoint-allowlist-macos:
43+
sha256: 1234
44+
size: 2
45+
url: /relative/path/to/endpoint-allowlist-macos
46+
- endpoint-allowlist-linux:
47+
sha256: 1234
48+
size: 2
49+
url: /relative/path/to/endpoint-allowlist-linux
50+
policy:
51+
linux:
52+
advanced:
53+
free-form: free-form-value
54+
indices:
55+
network: logs-endpoint.events.network-default
56+
file: logs-endpoint.events.file-default
57+
process: logs-endpoint.events.process-default
58+
metadata: metrics-endpoint.metadata-default
59+
policy: metrics-endpoint.policy-default
60+
telemetry: metrics-endpoint.telemetry-default
61+
logging:
62+
file: info
63+
stdout: debug
64+
events:
65+
process: true
66+
file: true
67+
network: true
68+
windows:
69+
malware:
70+
mode: prevent
71+
advanced:
72+
free-form: free-form-value
73+
indices:
74+
network: logs-endpoint.events.network-default
75+
file: logs-endpoint.events.file-default
76+
registry: logs-endpoint.events.registry-default
77+
process: logs-endpoint.events.process-default
78+
driver: logs-endpoint.events.driver-default
79+
library: logs-endpoint.events.library-default
80+
alerts: logs-endpoint.alerts-default
81+
metadata: metrics-endpoint.metadata-default
82+
policy: metrics-endpoint.policy-default
83+
telemetry: metrics-endpoint.telemetry-default
84+
logging:
85+
file: info
86+
stdout: debug
87+
events:
88+
registry: true
89+
process: true
90+
security: true
91+
file: true
92+
dns: false
93+
dll_and_driver_load: false
94+
network: true
95+
mac:
96+
malware:
97+
mode: prevent
98+
advanced:
99+
free-form: free-form-value
100+
indices:
101+
network: logs-endpoint.events.network-default
102+
file: logs-endpoint.events.file-default
103+
process: logs-endpoint.events.process-default
104+
alerts: logs-endpoint.alerts-default
105+
metadata: metrics-endpoint.metadata-default
106+
policy: metrics-endpoint.policy-default
107+
telemetry: metrics-endpoint.telemetry-default
108+
logging:
109+
file: info
110+
stdout: debug
111+
events:
112+
process: true
113+
file: true
114+
network: true

‎x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
filebeat:
2+
inputs:
3+
- type: log
4+
paths:
5+
- /var/log/hello1.log
6+
- /var/log/hello2.log
7+
index: logs-generic-default
8+
vars:
9+
var: value
10+
processors:
11+
- add_fields:
12+
target: "data_stream"
13+
fields:
14+
type: logs
15+
dataset: generic
16+
namespace: default
17+
- add_fields:
18+
target: "event"
19+
fields:
20+
dataset: generic
21+
- add_fields:
22+
target: "elastic_agent"
23+
fields:
24+
id: agent-id
25+
version: 8.0.0
26+
snapshot: false
27+
- add_fields:
28+
target: "agent"
29+
fields:
30+
id: agent-id
31+
- type: log
32+
paths:
33+
- /var/log/hello3.log
34+
- /var/log/hello4.log
35+
index: testtype-generic-default
36+
vars:
37+
var: value
38+
processors:
39+
- add_fields:
40+
target: "data_stream"
41+
fields:
42+
type: testtype
43+
dataset: generic
44+
namespace: default
45+
- add_fields:
46+
target: "event"
47+
fields:
48+
dataset: generic
49+
- add_fields:
50+
target: "elastic_agent"
51+
fields:
52+
id: agent-id
53+
version: 8.0.0
54+
snapshot: false
55+
- add_fields:
56+
target: "agent"
57+
fields:
58+
id: agent-id
59+
output:
60+
elasticsearch:
61+
hosts:
62+
- 127.0.0.1:9200
63+
- 127.0.0.1:9300
64+
namespace: test_namespace
65+
username: elastic
66+
password: changeme
67+
api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA
68+
ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y=

‎x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
fleet:
2+
agent:
3+
id: fleet-agent-id
4+
logging.level: error
5+
host:
6+
id: host-agent-id
7+
8+
output:
9+
elasticsearch:
10+
hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ]
11+
username: fleet
12+
password: fleetpassword
13+
14+
inputs:
15+
- id: fleet-server-id
16+
type: fleet-server

0 commit comments

Comments
 (0)