Skip to content

Add k8s cluster identifier #26056

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 34 commits into from
Jun 16, 2021
Merged

Add k8s cluster identifier #26056

Show file tree
Hide file tree
Changes from 26 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
480d046
wip
ChrsMark May 27, 2021
f301562
Refactor and add orchestrator.cluster.url
ChrsMark Jun 1, 2021
5a697b5
Merge remote-tracking branch 'upstream/master' into add_k8s_cluster_i…
ChrsMark Jun 1, 2021
54994fb
wip
ChrsMark Jun 4, 2021
c2555dd
Update methods to get cluster Identifiers
ChrsMark Jun 4, 2021
fd6d2e6
Remove unused function
ChrsMark Jun 4, 2021
5e75367
wip
ChrsMark Jun 7, 2021
2df343a
wip
ChrsMark Jun 7, 2021
943502e
wip
ChrsMark Jun 7, 2021
b960f84
wip
ChrsMark Jun 7, 2021
1bfd7ea
wip
ChrsMark Jun 7, 2021
2b5c55a
stable commit
ChrsMark Jun 7, 2021
bebec37
fixups
ChrsMark Jun 7, 2021
3facd8a
remove unused changes
ChrsMark Jun 7, 2021
bbd7b70
Add configmaps in clusterrole
ChrsMark Jun 8, 2021
3b2761f
Tune add_k8s_metadata processor
ChrsMark Jun 8, 2021
25fd5a5
Merge remote-tracking branch 'upstream/master' into add_k8s_cluster_i…
ChrsMark Jun 8, 2021
744afe4
cleanup
ChrsMark Jun 8, 2021
8429489
Move GKE metadata code to cloud metadata lib
ChrsMark Jun 8, 2021
85b9a1e
fix struct
ChrsMark Jun 8, 2021
34747af
Add orchestator.cluster.name only if in k8s
ChrsMark Jun 9, 2021
8a8c6d1
Improve cloud_metadata
ChrsMark Jun 9, 2021
018c7ea
Fix k8s roles
ChrsMark Jun 9, 2021
4103b87
Fix metricsets
ChrsMark Jun 9, 2021
e6f41b4
Fixup
ChrsMark Jun 9, 2021
d1c9ef6
fmt
ChrsMark Jun 9, 2021
04697d1
review changes
ChrsMark Jun 10, 2021
c06926a
Add changelog
ChrsMark Jun 10, 2021
503436c
Merge remote-tracking branch 'upstream/master' into add_k8s_cluster_i…
ChrsMark Jun 16, 2021
eb115c9
fixes
ChrsMark Jun 16, 2021
5a03920
fix expected files
ChrsMark Jun 16, 2021
1ee92c8
fix tests
ChrsMark Jun 16, 2021
c69f878
small fix
ChrsMark Jun 16, 2021
e720e74
formating
ChrsMark Jun 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions deploy/kubernetes/auditbeat-kubernetes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,20 @@ roleRef:
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: auditbeat
namespace: kube-system
subjects:
- kind: ServiceAccount
name: auditbeat
namespace: kube-system
roleRef:
kind: Role
name: auditbeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: auditbeat
Expand All @@ -243,6 +257,24 @@ rules:
- replicasets
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: auditbeat
namespace: kube-system
labels:
k8s-app: auditbeat
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs: ["get", "create", "update"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
Expand Down
14 changes: 14 additions & 0 deletions deploy/kubernetes/auditbeat/auditbeat-role-binding.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,17 @@ roleRef:
kind: ClusterRole
name: auditbeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: auditbeat
namespace: kube-system
subjects:
- kind: ServiceAccount
name: auditbeat
namespace: kube-system
roleRef:
kind: Role
name: auditbeat
apiGroup: rbac.authorization.k8s.io
18 changes: 18 additions & 0 deletions deploy/kubernetes/auditbeat/auditbeat-role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,21 @@ rules:
resources:
- replicasets
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: auditbeat
namespace: kube-system
labels:
k8s-app: auditbeat
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs: ["get", "create", "update"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]
1 change: 1 addition & 0 deletions deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,7 @@ rules:
- events
- pods
- services
- configmaps
verbs: ["get", "list", "watch"]
# Enable this rule only if planing to use kubernetes_secrets provider
#- apiGroups: [""]
Expand Down
1 change: 1 addition & 0 deletions deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ rules:
- events
- pods
- services
- configmaps
verbs: ["get", "list", "watch"]
# Enable this rule only if planing to use kubernetes_secrets provider
#- apiGroups: [""]
Expand Down
32 changes: 32 additions & 0 deletions deploy/kubernetes/filebeat-kubernetes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,20 @@ roleRef:
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: filebeat
namespace: kube-system
subjects:
- kind: ServiceAccount
name: filebeat
namespace: kube-system
roleRef:
kind: Role
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
Expand All @@ -161,6 +175,24 @@ rules:
- replicasets
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs: ["get", "create", "update"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
Expand Down
14 changes: 14 additions & 0 deletions deploy/kubernetes/filebeat/filebeat-role-binding.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,17 @@ roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: filebeat
namespace: kube-system
subjects:
- kind: ServiceAccount
name: filebeat
namespace: kube-system
roleRef:
kind: Role
name: filebeat
apiGroup: rbac.authorization.k8s.io
18 changes: 18 additions & 0 deletions deploy/kubernetes/filebeat/filebeat-role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,21 @@ rules:
resources:
- replicasets
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs: ["get", "create", "update"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]
32 changes: 32 additions & 0 deletions deploy/kubernetes/heartbeat-kubernetes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,20 @@ roleRef:
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: heartbeat
namespace: kube-system
subjects:
- kind: ServiceAccount
name: heartbeat
namespace: kube-system
roleRef:
kind: Role
name: heartbeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: heartbeat
Expand All @@ -154,6 +168,24 @@ rules:
- replicasets
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: heartbeat
namespace: kube-system
labels:
k8s-app: heartbeat
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs: ["get", "create", "update"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
Expand Down
14 changes: 14 additions & 0 deletions deploy/kubernetes/heartbeat/heartbeat-role-binding.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,17 @@ roleRef:
kind: ClusterRole
name: heartbeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: heartbeat
namespace: kube-system
subjects:
- kind: ServiceAccount
name: heartbeat
namespace: kube-system
roleRef:
kind: Role
name: heartbeat
apiGroup: rbac.authorization.k8s.io
18 changes: 18 additions & 0 deletions deploy/kubernetes/heartbeat/heartbeat-role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,21 @@ rules:
resources:
- replicasets
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: heartbeat
namespace: kube-system
labels:
k8s-app: heartbeat
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs: ["get", "create", "update"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]
4 changes: 4 additions & 0 deletions deploy/kubernetes/metricbeat-kubernetes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,10 @@ rules:
resources:
- leases
verbs: ["get", "create", "update"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
Expand Down
4 changes: 4 additions & 0 deletions deploy/kubernetes/metricbeat/metricbeat-role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,7 @@ rules:
resources:
- leases
verbs: ["get", "create", "update"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]
10 changes: 5 additions & 5 deletions libbeat/autodiscover/providers/kubernetes/node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func NewNodeEventer(uuid uuid.UUID, cfg *common.Config, client k8s.Interface, pu
config: config,
uuid: uuid,
publish: publish,
metagen: metadata.NewNodeMetadataGenerator(cfg, watcher.Store()),
metagen: metadata.NewNodeMetadataGenerator(cfg, watcher.Store(), client),
logger: logger,
watcher: watcher,
}
Expand Down Expand Up @@ -177,7 +177,9 @@ func (n *node) emit(node *kubernetes.Node, flag string) {
eventID := fmt.Sprint(node.GetObjectMeta().GetUID())
meta := n.metagen.Generate(node)

kubemeta := meta.Clone()
kubemetaMap, _ := meta.GetValue("kubernetes")
kubemeta, _ := kubemetaMap.(common.MapStr)
kubemeta = kubemeta.Clone()
// Pass annotations to all events so that it can be used in templating and by annotation builders.
annotations := common.MapStr{}
for k, v := range node.GetObjectMeta().GetAnnotations() {
Expand All @@ -190,9 +192,7 @@ func (n *node) emit(node *kubernetes.Node, flag string) {
flag: true,
"host": host,
"kubernetes": kubemeta,
"meta": common.MapStr{
"kubernetes": meta,
},
"meta": meta,
}
n.publish([]bus.Event{event})
}
Expand Down
4 changes: 3 additions & 1 deletion libbeat/autodiscover/providers/kubernetes/node_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
k8sfake "k8s.io/client-go/kubernetes/fake"

"github.com/elastic/beats/v7/libbeat/autodiscover/template"
"github.com/elastic/beats/v7/libbeat/common"
Expand Down Expand Up @@ -111,6 +112,7 @@ func TestGenerateHints_Node(t *testing.T) {
}

func TestEmitEvent_Node(t *testing.T) {
client := k8sfake.NewSimpleClientset()
name := "metricbeat"
nodeIP := "192.168.0.1"
uid := "005f3b90-4b9d-12f8-acf0-31020a840133"
Expand Down Expand Up @@ -310,7 +312,7 @@ func TestEmitEvent_Node(t *testing.T) {
t.Fatal(err)
}

metaGen := metadata.NewNodeMetadataGenerator(common.NewConfig(), nil)
metaGen := metadata.NewNodeMetadataGenerator(common.NewConfig(), nil, client)
config := defaultConfig()
p := &Provider{
config: config,
Expand Down
Loading