From 2eecebac8599971596c442fd53d05baa49226564 Mon Sep 17 00:00:00 2001 From: Chad Patel Date: Tue, 3 Oct 2023 11:11:37 -0500 Subject: [PATCH] updates for enhanced container insights (#148) --- container-insights-manifest-update.sh | 4 ++-- .../cwagent-ecs-instance-metric-cfn.json | 2 +- .../cwagent-ecs-instance-metric.json | 2 +- ...gent-ecs-prometheus-metric-for-awsvpc.yaml | 2 +- ...ecs-prometheus-metric-for-bridge-host.yaml | 2 +- .../cwagent-prometheus-task-definition.json | 2 +- .../sidecar/combination/combination-ec2.json | 2 +- .../combination/combination-fargate.json | 2 +- .../sidecar/cwagent-emf/cwagent-emf-ec2.json | 2 +- .../cwagent-emf/cwagent-emf-fargate.json | 2 +- .../cwagent-sdkmetrics-ec2.json | 2 +- .../cwagent-sdkmetrics-fargate.json | 2 +- .../cwagent-statsd/cwagent-statsd-ec2.json | 2 +- .../cwagent-statsd-fargate.json | 2 +- .../daemonset/combination/combination.yaml | 17 ++++++++------ .../cwagent/cwagent-configmap.yaml | 3 ++- .../cwagent/cwagent-daemonset.yaml | 12 +++++----- .../cwagent/cwagent-serviceaccount.yaml | 4 +++- .../fluent-bit/fluent-bit-compatible.yaml | 2 +- .../fluent-bit/fluent-bit.yaml | 2 +- .../fluentd/fluentd.yaml | 2 +- .../cwagent-fluent-bit-quickstart.yaml | 21 +++++++++-------- .../cwagent-fluentd-quickstart.yaml | 23 +++++++++++-------- .../cwagent-fluentd-xray-quickstart.yaml | 2 +- .../cwagent-sdkmetrics.yaml | 2 +- .../cwagent-statsd/cwagent-statsd.yaml | 2 +- .../service/combination/combination.yaml | 2 +- .../prometheus-eks-fargate.yaml | 4 ++-- .../cwagent-prometheus/prometheus-eks.yaml | 4 ++-- .../cwagent-prometheus/prometheus-k8s.yaml | 4 ++-- .../cwagent-sdkmetrics.yaml | 2 +- .../cwagent-statsd/cwagent-statsd.yaml | 2 +- .../sidecar/combination/combination.yaml | 2 +- .../sidecar/cwagent-emf/cwagent-emf.yaml | 2 +- .../cwagent-sdkmetrics.yaml | 2 +- .../cwagent-statsd/cwagent-statsd.yaml | 2 +- .../cwagent-configmap.yaml | 3 ++- .../cwagent-daemonset.yaml | 10 ++++---- .../cwagent-serviceaccount.yaml | 4 +++- .../cwagent-statsd-daemonset.yaml | 2 +- .../cwagent-statsd-deployment.yaml | 2 +- .../cwagent-fluentd-quickstart.yaml | 17 ++++++++------ 42 files changed, 102 insertions(+), 84 deletions(-) diff --git a/container-insights-manifest-update.sh b/container-insights-manifest-update.sh index 2a462ca..ec30843 100755 --- a/container-insights-manifest-update.sh +++ b/container-insights-manifest-update.sh @@ -4,8 +4,8 @@ cd "$(dirname "$0")" k8sDirPrefix="./k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring" ecsDirPrefix="./ecs-task-definition-templates/deployment-mode/daemon-service/cwagent-ecs-instance-metric" -newK8sVersion="k8s/1.3.16" -agentVersion="public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.247360.0b252689" +newK8sVersion="k8s/1.3.17" +agentVersion="public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210" fluentdVersion="fluent/fluentd-kubernetes-daemonset:v1.10.3-debian-cloudwatch-1.0" fluentBitVersion="public.ecr.aws/aws-observability/aws-for-fluent-bit:stable" diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/cwagent-ecs-instance-metric/cloudformation-quickstart/cwagent-ecs-instance-metric-cfn.json b/ecs-task-definition-templates/deployment-mode/daemon-service/cwagent-ecs-instance-metric/cloudformation-quickstart/cwagent-ecs-instance-metric-cfn.json index bfe1cc8..b567e31 100644 --- a/ecs-task-definition-templates/deployment-mode/daemon-service/cwagent-ecs-instance-metric/cloudformation-quickstart/cwagent-ecs-instance-metric-cfn.json +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/cwagent-ecs-instance-metric/cloudformation-quickstart/cwagent-ecs-instance-metric-cfn.json @@ -105,7 +105,7 @@ "ContainerDefinitions": [ { "Name": "cloudwatch-agent", - "Image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.247360.0b252689", + "Image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "MountPoints": [ { "ReadOnly": true, diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/cwagent-ecs-instance-metric/cwagent-ecs-instance-metric.json b/ecs-task-definition-templates/deployment-mode/daemon-service/cwagent-ecs-instance-metric/cwagent-ecs-instance-metric.json index 6e14fc1..dc62ce7 100644 --- a/ecs-task-definition-templates/deployment-mode/daemon-service/cwagent-ecs-instance-metric/cwagent-ecs-instance-metric.json +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/cwagent-ecs-instance-metric/cwagent-ecs-instance-metric.json @@ -6,7 +6,7 @@ "containerDefinitions": [ { "name": "cloudwatch-agent", - "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.247360.0b252689", + "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "mountPoints": [ { "readOnly": true, diff --git a/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cloudformation-quickstart/cwagent-ecs-prometheus-metric-for-awsvpc.yaml b/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cloudformation-quickstart/cwagent-ecs-prometheus-metric-for-awsvpc.yaml index cf8d7be..201b350 100644 --- a/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cloudformation-quickstart/cwagent-ecs-prometheus-metric-for-awsvpc.yaml +++ b/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cloudformation-quickstart/cwagent-ecs-prometheus-metric-for-awsvpc.yaml @@ -224,7 +224,7 @@ Resources: NetworkMode: awsvpc ContainerDefinitions: - Name: cloudwatch-agent-prometheus - Image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.247360.0b252689 + Image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210 Essential: true MountPoints: [] PortMappings: [] diff --git a/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cloudformation-quickstart/cwagent-ecs-prometheus-metric-for-bridge-host.yaml b/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cloudformation-quickstart/cwagent-ecs-prometheus-metric-for-bridge-host.yaml index 1fe7586..ec10acd 100644 --- a/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cloudformation-quickstart/cwagent-ecs-prometheus-metric-for-bridge-host.yaml +++ b/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cloudformation-quickstart/cwagent-ecs-prometheus-metric-for-bridge-host.yaml @@ -219,7 +219,7 @@ Resources: NetworkMode: !Ref ECSNetworkMode ContainerDefinitions: - Name: cloudwatch-agent-prometheus - Image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.247360.0b252689 + Image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210 Essential: true MountPoints: [] PortMappings: [] diff --git a/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cwagent-prometheus-task-definition.json b/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cwagent-prometheus-task-definition.json index f4a1b8e..8b067d7 100644 --- a/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cwagent-prometheus-task-definition.json +++ b/ecs-task-definition-templates/deployment-mode/replica-service/cwagent-prometheus/cwagent-prometheus-task-definition.json @@ -6,7 +6,7 @@ "containerDefinitions": [ { "name": "cloudwatch-agent-prometheus", - "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.247360.0b252689", + "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "essential": true, "mountPoints": [ ], diff --git a/ecs-task-definition-templates/deployment-mode/sidecar/combination/combination-ec2.json b/ecs-task-definition-templates/deployment-mode/sidecar/combination/combination-ec2.json index 74eef2a..84da029 100644 --- a/ecs-task-definition-templates/deployment-mode/sidecar/combination/combination-ec2.json +++ b/ecs-task-definition-templates/deployment-mode/sidecar/combination/combination-ec2.json @@ -22,7 +22,7 @@ }, { "name": "cloudwatch-agent", - "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest", + "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "secrets": [ { "name": "CW_CONFIG_CONTENT", diff --git a/ecs-task-definition-templates/deployment-mode/sidecar/combination/combination-fargate.json b/ecs-task-definition-templates/deployment-mode/sidecar/combination/combination-fargate.json index c2e07cb..33cf557 100644 --- a/ecs-task-definition-templates/deployment-mode/sidecar/combination/combination-fargate.json +++ b/ecs-task-definition-templates/deployment-mode/sidecar/combination/combination-fargate.json @@ -19,7 +19,7 @@ }, { "name": "cloudwatch-agent", - "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest", + "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "secrets": [ { "name": "CW_CONFIG_CONTENT", diff --git a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-emf/cwagent-emf-ec2.json b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-emf/cwagent-emf-ec2.json index 74eef2a..84da029 100644 --- a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-emf/cwagent-emf-ec2.json +++ b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-emf/cwagent-emf-ec2.json @@ -22,7 +22,7 @@ }, { "name": "cloudwatch-agent", - "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest", + "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "secrets": [ { "name": "CW_CONFIG_CONTENT", diff --git a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-emf/cwagent-emf-fargate.json b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-emf/cwagent-emf-fargate.json index c2e07cb..33cf557 100644 --- a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-emf/cwagent-emf-fargate.json +++ b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-emf/cwagent-emf-fargate.json @@ -19,7 +19,7 @@ }, { "name": "cloudwatch-agent", - "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest", + "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "secrets": [ { "name": "CW_CONFIG_CONTENT", diff --git a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-sdkmetrics/cwagent-sdkmetrics-ec2.json b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-sdkmetrics/cwagent-sdkmetrics-ec2.json index c018ffa..127556f 100644 --- a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-sdkmetrics/cwagent-sdkmetrics-ec2.json +++ b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-sdkmetrics/cwagent-sdkmetrics-ec2.json @@ -22,7 +22,7 @@ }, { "name": "cloudwatch-agent", - "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest", + "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "secrets": [ { "name": "CW_CONFIG_CONTENT", diff --git a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-sdkmetrics/cwagent-sdkmetrics-fargate.json b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-sdkmetrics/cwagent-sdkmetrics-fargate.json index d4333cb..a9f389d 100644 --- a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-sdkmetrics/cwagent-sdkmetrics-fargate.json +++ b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-sdkmetrics/cwagent-sdkmetrics-fargate.json @@ -19,7 +19,7 @@ }, { "name": "cloudwatch-agent", - "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest", + "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "secrets": [ { "name": "CW_CONFIG_CONTENT", diff --git a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-statsd/cwagent-statsd-ec2.json b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-statsd/cwagent-statsd-ec2.json index 36da693..13f4932 100644 --- a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-statsd/cwagent-statsd-ec2.json +++ b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-statsd/cwagent-statsd-ec2.json @@ -27,7 +27,7 @@ }, { "name": "cloudwatch-agent", - "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest", + "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "secrets": [ { "name": "CW_CONFIG_CONTENT", diff --git a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-statsd/cwagent-statsd-fargate.json b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-statsd/cwagent-statsd-fargate.json index 757497b..8a380e3 100644 --- a/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-statsd/cwagent-statsd-fargate.json +++ b/ecs-task-definition-templates/deployment-mode/sidecar/cwagent-statsd/cwagent-statsd-fargate.json @@ -24,7 +24,7 @@ }, { "name": "cloudwatch-agent", - "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest", + "image": "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210", "secrets": [ { "name": "CW_CONFIG_CONTENT", diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml index 66ec175..9df0f6b 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml @@ -24,7 +24,7 @@ rules: resources: ["pods", "nodes", "endpoints"] verbs: ["list", "watch"] - apiGroups: ["apps"] - resources: ["replicasets"] + resources: ["replicasets", "daemonsets", "deployments"] verbs: ["list", "watch"] - apiGroups: ["batch"] resources: ["jobs"] @@ -39,6 +39,8 @@ rules: resources: ["configmaps"] resourceNames: ["cwagent-clusterleader"] verbs: ["get","update"] + - nonResourceURLs: ["/metrics"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding @@ -71,7 +73,8 @@ data: "metrics_collected": { "kubernetes": { "cluster_name": "{{cluster_name}}", - "metrics_collection_interval": 60 + "metrics_collection_interval": 60, + "enhanced_container_insights": true }, "emf": {} }, @@ -113,7 +116,7 @@ spec: spec: containers: - name: cloudwatch-agent - image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest + image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210 ports: - containerPort: 8125 hostPort: 8125 @@ -126,11 +129,11 @@ spec: protocol: UDP resources: limits: - cpu: 200m - memory: 200Mi + cpu: 400m + memory: 400Mi requests: - cpu: 200m - memory: 200Mi + cpu: 400m + memory: 400Mi # Please don't change below envs env: - name: HOST_IP diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-configmap.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-configmap.yaml index 58fbdd7..5c210d2 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-configmap.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-configmap.yaml @@ -9,7 +9,8 @@ data: "metrics_collected": { "kubernetes": { "cluster_name": "{{cluster_name}}", - "metrics_collection_interval": 60 + "metrics_collection_interval": 60, + "enhanced_container_insights": true } }, "force_flush_interval": 5 diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml index 24d9047..bf94dc0 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml @@ -15,18 +15,18 @@ spec: spec: containers: - name: cloudwatch-agent - image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.247360.0b252689 + image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300028.1b210 #ports: # - containerPort: 8125 # hostPort: 8125 # protocol: UDP resources: limits: - cpu: 200m - memory: 200Mi + cpu: 400m + memory: 400Mi requests: - cpu: 200m - memory: 200Mi + cpu: 400m + memory: 400Mi # Please don't change below envs env: - name: HOST_IP @@ -42,7 +42,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CI_VERSION - value: "k8s/1.3.16" + value: "k8s/1.3.17" # Please don't change the mountPath volumeMounts: - name: cwagentconfig diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-serviceaccount.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-serviceaccount.yaml index cf4e699..7c62cc5 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-serviceaccount.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-serviceaccount.yaml @@ -15,7 +15,7 @@ rules: resources: ["pods", "nodes", "endpoints"] verbs: ["list", "watch"] - apiGroups: ["apps"] - resources: ["replicasets"] + resources: ["replicasets", "daemonsets", "deployments"] verbs: ["list", "watch"] - apiGroups: ["batch"] resources: ["jobs"] @@ -30,6 +30,8 @@ rules: resources: ["configmaps"] resourceNames: ["cwagent-clusterleader"] verbs: ["get","update"] + - nonResourceURLs: ["/metrics"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit-compatible.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit-compatible.yaml index 777975a..e55a73b 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit-compatible.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit-compatible.yaml @@ -320,7 +320,7 @@ spec: fieldRef: fieldPath: spec.nodeName - name: CI_VERSION - value: "k8s/1.3.16" + value: "k8s/1.3.17" resources: limits: memory: 200Mi diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit.yaml index 2668ef8..9844f45 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit.yaml @@ -304,7 +304,7 @@ spec: apiVersion: v1 fieldPath: metadata.name - name: CI_VERSION - value: "k8s/1.3.16" + value: "k8s/1.3.17" resources: limits: memory: 200Mi diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml index 4bdf80b..9771512 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml @@ -398,7 +398,7 @@ spec: name: cluster-info key: cluster.name - name: CI_VERSION - value: "k8s/1.3.16" + value: "k8s/1.3.17" - name: FLUENT_CONTAINER_TAIL_PARSER_TYPE value: /^(?