From 7bba032ab3ffa45de74b32fb87b0ea4cfd7d8b21 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Sat, 28 Jun 2025 15:04:27 -0700
Subject: [PATCH 1/6] ta: inline tls config for config generation

fixes: #3724

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .../internal/watcher/promOperator.go          |   2 +-
 .../internal/watcher/promOperator_test.go     |   2 +-
 .../00-assert.yaml                            |  16 ++
 .../00-install.yaml                           | 211 ++++++++++++++++++
 .../01-assert.yaml                            |  20 ++
 .../01-install.yaml                           |  60 +++++
 .../chainsaw-test.yaml                        |  72 ++++++
 7 files changed, 381 insertions(+), 2 deletions(-)
 create mode 100644 tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-assert.yaml
 create mode 100644 tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-install.yaml
 create mode 100644 tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-assert.yaml
 create mode 100644 tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-install.yaml
 create mode 100755 tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml

diff --git a/cmd/otel-allocator/internal/watcher/promOperator.go b/cmd/otel-allocator/internal/watcher/promOperator.go
index 4ea16f7b72..6110aec68d 100644
--- a/cmd/otel-allocator/internal/watcher/promOperator.go
+++ b/cmd/otel-allocator/internal/watcher/promOperator.go
@@ -89,7 +89,7 @@ func NewPrometheusCRWatcher(ctx context.Context, logger logr.Logger, cfg allocat
 		},
 	}
 
-	generator, err := prometheus.NewConfigGenerator(promLogger, prom, prometheus.WithEndpointSliceSupport())
+	generator, err := prometheus.NewConfigGenerator(promLogger, prom, prometheus.WithEndpointSliceSupport(), prometheus.WithInlineTLSConfig())
 
 	if err != nil {
 		return nil, err
diff --git a/cmd/otel-allocator/internal/watcher/promOperator_test.go b/cmd/otel-allocator/internal/watcher/promOperator_test.go
index d284d2403e..7ce879cb11 100644
--- a/cmd/otel-allocator/internal/watcher/promOperator_test.go
+++ b/cmd/otel-allocator/internal/watcher/promOperator_test.go
@@ -1251,7 +1251,7 @@ func getTestPrometheusCRWatcher(
 
 	promOperatorLogger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
 
-	generator, err := prometheus.NewConfigGenerator(promOperatorLogger, prom, prometheus.WithEndpointSliceSupport())
+	generator, err := prometheus.NewConfigGenerator(promOperatorLogger, prom, prometheus.WithEndpointSliceSupport(), prometheus.WithInlineTLSConfig())
 	if err != nil {
 		t.Fatal(t, err)
 	}
diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-assert.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-assert.yaml
new file mode 100644
index 0000000000..d8e25b41eb
--- /dev/null
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-assert.yaml
@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: prometheus-cr-v1beta1-collector
+status:
+  readyReplicas: 1
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-cr-v1beta1-targetallocator
+status:
+  observedGeneration: 1
+  readyReplicas: 1
+  replicas: 1
diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-install.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-install.yaml
new file mode 100644
index 0000000000..9c8b8a36ee
--- /dev/null
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-install.yaml
@@ -0,0 +1,211 @@
+apiVersion: v1
+automountServiceAccountToken: true
+kind: ServiceAccount
+metadata:
+  name: ta
+---
+apiVersion: v1
+automountServiceAccountToken: true
+kind: ServiceAccount
+metadata:
+  name: collector
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: collector
+  annotations:
+    kubernetes.io/service-account.name: collector
+type: kubernetes.io/service-account-token
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: (join('-', ['ta', $namespace]))
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - services
+  - endpoints
+  - configmaps
+  - secrets
+  - namespaces
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  - services
+  - endpoints
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  - podmonitors
+  - scrapeconfigs
+  - probes
+  verbs:
+  - get
+  - watch
+  - list
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: (join('-', ['collector', $namespace]))
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - nodes/proxy
+  - nodes/metrics
+  - services
+  - endpoints
+  - namespaces
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - watch
+  - list
+- nonResourceURLs:
+  - /metrics
+  - /metrics/cadvisor
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name:  (join('-', ['opentelemetry-operator-controller-manager-cert-manager-role', $namespace]))
+rules:
+- apiGroups:
+  - cert-manager.io
+  resources:
+  - issuers
+  - certificaterequests
+  - certificates
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: (join('-', ['opentelemetry-operator-controller-manager-cert-manager-role', $namespace]))
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: (join('-', ['opentelemetry-operator-controller-manager-cert-manager-role', $namespace]))
+subjects:
+- kind: ServiceAccount
+  name: opentelemetry-operator-controller-manager
+  namespace: opentelemetry-operator-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: (join('-', ['ta', $namespace]))
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: (join('-', ['ta', $namespace]))
+subjects:
+- kind: ServiceAccount
+  name: ta
+  namespace: ($namespace)
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: (join('-', ['collector', $namespace]))
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: (join('-', ['collector', $namespace]))
+subjects:
+- kind: ServiceAccount
+  name: collector
+  namespace: ($namespace)
+---
+apiVersion: opentelemetry.io/v1beta1
+kind: OpenTelemetryCollector
+metadata:
+  name: prometheus-cr-v1beta1
+spec:
+  config:
+    receivers:
+      prometheus:
+        config:
+          scrape_configs: []
+
+    processors:
+
+    exporters:
+      prometheus:
+        endpoint: 0.0.0.0:9090
+    service:
+      pipelines:
+        metrics:
+          receivers: [prometheus]
+          exporters: [prometheus]
+      telemetry:
+        metrics:
+          address: 0.0.0.0:8888
+          level: detailed
+        logs:
+          level: "DEBUG"
+          development: true
+          encoding: "json"
+  mode: statefulset
+  serviceAccount: collector
+  targetAllocator:
+    enabled: true
+    prometheusCR:
+      enabled: true
+      scrapeInterval: 1s
+      scrapeConfigSelector: {}
+    serviceAccount: ta
diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-assert.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-assert.yaml
new file mode 100644
index 0000000000..7b16f8a457
--- /dev/null
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-assert.yaml
@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-ta-scrape-config-v1beta1
+status:
+  succeeded: 1
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-collector-scrape-config-v1beta1
+status:
+  succeeded: 1
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-ta-jobs-scrape-configs-v1beta1
+status:
+  succeeded: 1
diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-install.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-install.yaml
new file mode 100644
index 0000000000..19014c46c2
--- /dev/null
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-install.yaml
@@ -0,0 +1,60 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-collector-scrape-config-v1beta1
+spec:
+  template:
+    metadata:
+      labels:
+        checker: "true"
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: check-metrics
+          image: curlimages/curl
+          env:
+            - name: namespace
+              value: ($namespace)
+          args:
+            - /bin/sh
+            - -c
+            - |
+              curl -s http://prometheus-cr-v1beta1-collector:9090/metrics | grep 'up{' | egrep "scrapeConfig/$namespace/scrape-config-cr" | egrep "1$"
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-ta-scrape-config-v1beta1
+spec:
+  template:
+    metadata:
+      labels:
+        checker: "true"
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: check-metrics
+          image: curlimages/curl
+          args:
+            - /bin/sh
+            - -c
+            - curl -s http://prometheus-cr-v1beta1-targetallocator/scrape_configs | grep "scrape-config-cr"
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-ta-jobs-scrape-configs-v1beta1
+spec:
+  template:
+    metadata:
+      labels:
+        checker: "true"
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: check-metrics
+          image: curlimages/curl
+          args:
+            - /bin/sh
+            - -c
+            - curl -s http://prometheus-cr-v1beta1-targetallocator/jobs | grep "scrape-config-cr"
diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml
new file mode 100755
index 0000000000..8822d7e68b
--- /dev/null
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml
@@ -0,0 +1,72 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: ta-collector-mtls-scrapeconfig-node
+spec:
+  steps:
+  - name: step-00
+    try:
+    - apply:
+        file: 00-install.yaml
+    - assert:
+        file: 00-assert.yaml
+    catch:
+    - podLogs:
+        selector: app.kubernetes.io/managed-by=opentelemetry-operator
+    - podLogs:
+        selector: checker=true
+  - name: step-01
+    try:
+      - create:
+          template: false
+          resource:
+            apiVersion: monitoring.coreos.com/v1alpha1
+            kind: ScrapeConfig
+            metadata:
+              name: scrape-config-cr
+            spec:
+              kubernetesSDConfigs:
+                - role: Node
+              relabelings:
+                - action: labelmap
+                  regex: __meta_kubernetes_node_label_(.+)
+                  replacement: $1
+                  separator: ;
+                - action: replace
+                  regex: (.*)
+                  replacement: kubernetes.default.svc:443
+                  separator: ;
+                  targetLabel: __address__
+                - action: replace
+                  regex: (.+)
+                  replacement: /api/v1/nodes/$1/proxy/metrics
+                  separator: ;
+                  sourceLabels:
+                    - __meta_kubernetes_node_name
+                  targetLabel: __metrics_path__
+              scheme: HTTPS
+              authorization:
+                type: Bearer
+                credentials:
+                  key: "token"
+                  name: "collector"
+              tlsConfig:
+                ca:
+                  secret:
+                    key: "ca.crt"
+                    name: "collector"
+                insecureSkipVerify: false
+      - apply:
+          template: true
+          bindings:
+            - name: namespace
+              value: ($namespace)
+          file: 01-install.yaml
+      - assert:
+          file: 01-assert.yaml
+    catch:
+      - podLogs:
+          selector: app.kubernetes.io/managed-by=opentelemetry-operator
+      - podLogs:
+          selector: checker=true

From 13b83f0b16b1320984c48ee199f2fbc379486249 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Sat, 28 Jun 2025 15:10:44 -0700
Subject: [PATCH 2/6] add changelog

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .chloggen/fix-ta-tlsconfig-scrapeconfig.yaml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 .chloggen/fix-ta-tlsconfig-scrapeconfig.yaml

diff --git a/.chloggen/fix-ta-tlsconfig-scrapeconfig.yaml b/.chloggen/fix-ta-tlsconfig-scrapeconfig.yaml
new file mode 100644
index 0000000000..a24a77a6f0
--- /dev/null
+++ b/.chloggen/fix-ta-tlsconfig-scrapeconfig.yaml
@@ -0,0 +1,17 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
+component: target allocator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Allow collector to use TLS Config from Target Allocator with ScrapeConfig
+
+# One or more tracking issues related to the change
+issues: [3724]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  This change allows the target allocator to configure TLS Config for a collector using the ScrapeConfig.

From 7b33760a36e15f49ad4180cad160392add3baca4 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Sat, 28 Jun 2025 20:41:57 -0700
Subject: [PATCH 3/6] retry getting ta metrics

ta could come up after the collector which could cause it to go into a
clbo. this race condition can make the metrics endpoint unavailable and
cause the test to be flaky.

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .../targetallocator-metrics/chainsaw-test.yaml                | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/e2e-targetallocator/targetallocator-metrics/chainsaw-test.yaml b/tests/e2e-targetallocator/targetallocator-metrics/chainsaw-test.yaml
index b2b23fd62b..e1d93ad7bd 100644
--- a/tests/e2e-targetallocator/targetallocator-metrics/chainsaw-test.yaml
+++ b/tests/e2e-targetallocator/targetallocator-metrics/chainsaw-test.yaml
@@ -45,7 +45,9 @@ spec:
                 value: ($podName)
             content: |
               #!/bin/bash
-              kubectl get --raw /api/v1/namespaces/${NAMESPACE}/pods/${podName}:9090/proxy/metrics
+              while ! kubectl get --raw /api/v1/namespaces/${NAMESPACE}/pods/${podName}:9090/proxy/metrics; do
+                sleep 1
+              done
             outputs:
               - name: metrics
                 value: (x_metrics_decode($stdout))

From eb2e7299f01b0701aa843e501c47bac9456fb442 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Sun, 29 Jun 2025 09:40:45 -0700
Subject: [PATCH 4/6] improve testing conditions for less flakiness

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .../00-assert.yaml                                   | 10 +++++-----
 .../targetallocator-metrics/chainsaw-test.yaml       | 12 +++++-------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-assert.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-assert.yaml
index d8e25b41eb..81bed598f8 100644
--- a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-assert.yaml
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-assert.yaml
@@ -1,16 +1,16 @@
 apiVersion: apps/v1
-kind: StatefulSet
+kind: Deployment
 metadata:
-  name: prometheus-cr-v1beta1-collector
+  name: prometheus-cr-v1beta1-targetallocator
 status:
+  observedGeneration: 1
   readyReplicas: 1
   replicas: 1
 ---
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
-  name: prometheus-cr-v1beta1-targetallocator
+  name: prometheus-cr-v1beta1-collector
 status:
-  observedGeneration: 1
   readyReplicas: 1
   replicas: 1
diff --git a/tests/e2e-targetallocator/targetallocator-metrics/chainsaw-test.yaml b/tests/e2e-targetallocator/targetallocator-metrics/chainsaw-test.yaml
index e1d93ad7bd..560658aa7c 100644
--- a/tests/e2e-targetallocator/targetallocator-metrics/chainsaw-test.yaml
+++ b/tests/e2e-targetallocator/targetallocator-metrics/chainsaw-test.yaml
@@ -10,17 +10,15 @@ spec:
             file: 00-install.yaml
     - name: Wait for OpenTelemetryCollector and Target Allocator pods to be ready
       try:
-        - sleep:
-            duration: 10s 
         - wait:
-            apiVersion: v1
-            kind: Pod
+            apiVersion: opentelemetry.io/v1beta1
+            kind: OpenTelemetryCollector
             namespace: ($namespace)
             timeout: 1m
             for:
-              condition:
-                name: ready
-                value: 'true'
+              jsonPath:
+                path: '{.status.scale.statusReplicas}'
+                value: '1/1'
         - assert:
             file: 00-assert.yaml
     - name: Assert Target Allocator metrics

From d3e5f0635a99405b1f9c205c37f4630ec7965181 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Mon, 30 Jun 2025 21:55:11 -0700
Subject: [PATCH 5/6] use x_metrics_decode for parsing metric value

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .../00-install.yaml                           |  3 ---
 .../01-assert.yaml                            |  7 ------
 .../01-install.yaml                           | 23 -------------------
 .../chainsaw-test.yaml                        | 23 +++++++++++++++++++
 4 files changed, 23 insertions(+), 33 deletions(-)

diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-install.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-install.yaml
index 9c8b8a36ee..72f6ae7e1f 100644
--- a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-install.yaml
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/00-install.yaml
@@ -193,9 +193,6 @@ spec:
           receivers: [prometheus]
           exporters: [prometheus]
       telemetry:
-        metrics:
-          address: 0.0.0.0:8888
-          level: detailed
         logs:
           level: "DEBUG"
           development: true
diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-assert.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-assert.yaml
index 7b16f8a457..43042eb80b 100644
--- a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-assert.yaml
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-assert.yaml
@@ -7,13 +7,6 @@ status:
 ---
 apiVersion: batch/v1
 kind: Job
-metadata:
-  name: check-collector-scrape-config-v1beta1
-status:
-  succeeded: 1
----
-apiVersion: batch/v1
-kind: Job
 metadata:
   name: check-ta-jobs-scrape-configs-v1beta1
 status:
diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-install.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-install.yaml
index 19014c46c2..ec24f4f05b 100644
--- a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-install.yaml
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/01-install.yaml
@@ -1,28 +1,5 @@
 apiVersion: batch/v1
 kind: Job
-metadata:
-  name: check-collector-scrape-config-v1beta1
-spec:
-  template:
-    metadata:
-      labels:
-        checker: "true"
-    spec:
-      restartPolicy: OnFailure
-      containers:
-        - name: check-metrics
-          image: curlimages/curl
-          env:
-            - name: namespace
-              value: ($namespace)
-          args:
-            - /bin/sh
-            - -c
-            - |
-              curl -s http://prometheus-cr-v1beta1-collector:9090/metrics | grep 'up{' | egrep "scrapeConfig/$namespace/scrape-config-cr" | egrep "1$"
----
-apiVersion: batch/v1
-kind: Job
 metadata:
   name: check-ta-scrape-config-v1beta1
 spec:
diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml
index 8822d7e68b..6209ea423a 100755
--- a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml
@@ -65,6 +65,29 @@ spec:
           file: 01-install.yaml
       - assert:
           file: 01-assert.yaml
+      - script:
+          timeout: 1m
+          content: |
+            #!/bin/bash
+            # Wait for the scrape config to be used by the collector to scrape nodes. It can take 30s for the collector to start scraping nodes.
+            while ! kubectl get --raw /api/v1/namespaces/${NAMESPACE}/pods/prometheus-cr-v1beta1-collector-0:9090/proxy/metrics | grep up{ | grep "job=\"scrapeConfig/${NAMESPACE}/scrape-config-cr\""; do
+              sleep 1
+            done
+          outputs:
+            - name: metrics
+              value: ($stdout)
+          check:
+            ($error == null): true
+      - script:
+          env:
+            - name: TT
+              value: (to_string(x_metrics_decode($metrics)))
+          content: echo $TT
+      - assert:
+          timeout: 1s
+          resource:
+            ((x_metrics_decode($metrics))):
+              ([?as_string(metric.__name__) == 'up' && as_string(metric.job) == join('/', ['scrapeConfig', $namespace, 'scrape-config-cr'])].value): [ 1 ]
     catch:
       - podLogs:
           selector: app.kubernetes.io/managed-by=opentelemetry-operator

From 5493968825999b6a834260bac5dbb1b6e32887ed Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Tue, 1 Jul 2025 08:40:31 -0700
Subject: [PATCH 6/6] Update chainsaw-test.yaml

---
 .../ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml   | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml
index 6209ea423a..e7028fa857 100755
--- a/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml
+++ b/tests/e2e-ta-collector-mtls/ta-collector-mtls-scrapeconfig-node/chainsaw-test.yaml
@@ -78,11 +78,6 @@ spec:
               value: ($stdout)
           check:
             ($error == null): true
-      - script:
-          env:
-            - name: TT
-              value: (to_string(x_metrics_decode($metrics)))
-          content: echo $TT
       - assert:
           timeout: 1s
           resource: