Stop implicitly validating generic.otel for otelcol input packages

internal/kibana/packagepolicy.go

@@ -161,7 +161,12 @@ func BuildInputPackagePolicy(
 	}
 
 	// Dataset key for the stream: <package name>.<policy template name>.
+	// For packages with dynamic_signal_types, Fleet uses "generic.otel" as the
+	// stream path, so we must match that key in the simplified API request.
 	streamDataset := fmt.Sprintf("%s.%s", manifest.Name, policyTemplate.Name)
+	if policyTemplate.DynamicSignalTypes {
+		streamDataset = "generic.otel"
+	}
 
 	vars := SetKibanaVariables(policyTemplate.Vars, varValues)
 	ensureDatasetVar(vars, policyTemplate, varValues)
@@ -235,6 +240,9 @@ func ensureDatasetVar(vars Vars, policyTemplate packages.PolicyTemplate, varValu
 		return
 	}
 	dataset := policyTemplate.Name
+	if policyTemplate.DynamicSignalTypes {
+		dataset = "generic.otel"
+	}
 	for _, def := range policyTemplate.Vars {
 		if def.Name == "data_stream.dataset" && def.Default != nil {
 			if s, ok := def.Default.Value().(string); ok && s != "" {

internal/kibana/packagepolicy_test.go

@@ -282,6 +282,19 @@ func TestBuildInputPackagePolicy(t *testing.T) {
 			goldenSimplified: "testdata/otel_traces_use_apm.json",
 			goldenLegacy:     "testdata/otel_traces_use_apm_legacy.json",
 		},
+		{
+			// Input package with dynamic_signal_types: true. Fleet now uses
+			// "generic.otel" as the stream path/dataset for such packages, so
+			// BuildInputPackagePolicy must use "generic.otel" as the stream key
+			// and the data_stream.dataset variable value.
+			name:               "otel_input_dynamic_signals",
+			packageRoot:        "testdata/packages/otel_dynamic_input",
+			policyTemplateName: "sqlreceiver",
+			policyName:         "otel-dynamic-test",
+			varValues:          common.MapStr{},
+			goldenSimplified:   "testdata/otel_input_dynamic_signals.json",
+			goldenLegacy:       "testdata/otel_input_dynamic_signals_legacy.json",
+		},
 		{
 			// Package-level variable: the user overrides the default package-level
 			// var (custom_tag). BuildInputPackagePolicy must forward manifest.Vars
@@ -515,6 +528,13 @@ func TestEnsureDatasetVar(t *testing.T) {
 			varValues:      common.MapStr{},
 			wantDataset:    "sql_query",
 		},
+		{
+			name:           "dynamic_signal_types uses generic.otel",
+			vars:           Vars{},
+			policyTemplate: packages.PolicyTemplate{Name: "sqlreceiver", DynamicSignalTypes: true},
+			varValues:      common.MapStr{},
+			wantDataset:    "generic.otel",
+		},
 	}
 
 	for _, c := range cases {

internal/kibana/testdata/otel_input_dynamic_signals.json

@@ -0,0 +1,24 @@
+{
+  "name": "otel-dynamic-test",
+  "description": "",
+  "namespace": "test",
+  "policy_id": "test-policy-id",
+  "package": {
+    "name": "otel_dynamic_input",
+    "version": "0.1.0"
+  },
+  "inputs": {
+    "sqlreceiver-otelcol": {
+      "enabled": true,
+      "streams": {
+        "generic.otel": {
+          "enabled": true,
+          "vars": {
+            "data_stream.dataset": "generic.otel"
+          }
+        }
+      }
+    }
+  },
+  "force": false
+}

internal/kibana/testdata/otel_input_dynamic_signals_legacy.json

@@ -0,0 +1,42 @@
+{
+  "name": "otel-dynamic-test",
+  "description": "",
+  "namespace": "test",
+  "policy_id": "test-policy-id",
+  "enabled": true,
+  "package": {
+    "name": "otel_dynamic_input",
+    "title": "OTel Dynamic Signals Input Package",
+    "version": "0.1.0"
+  },
+  "inputs": [
+    {
+      "policy_template": "sqlreceiver",
+      "type": "otelcol",
+      "enabled": true,
+      "vars": {
+        "data_stream.dataset": {
+          "value": "generic.otel",
+          "type": "text"
+        }
+      },
+      "streams": [
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "",
+            "dataset": "generic.otel"
+          },
+          "vars": {
+            "data_stream.dataset": {
+              "value": "generic.otel",
+              "type": "text"
+            }
+          }
+        }
+      ]
+    }
+  ],
+  "output_id": "",
+  "force": false
+}

internal/kibana/testdata/packages/otel_dynamic_input/manifest.yml

@@ -0,0 +1,23 @@
+format_version: 3.5.0
+name: otel_dynamic_input
+title: OTel Dynamic Signals Input Package
+description: >-
+  Minimal OTel input package with dynamic_signal_types used to verify that
+  BuildInputPackagePolicy uses "generic.otel" as the stream key.
+type: input
+version: 0.1.0
+categories:
+  - observability
+conditions:
+  kibana:
+    version: "^9.4.0"
+policy_templates:
+  - name: sqlreceiver
+    title: SQL Receiver
+    description: Collect SQL metrics and logs using OTel.
+    input: otelcol
+    template_path: input.yml.hbs
+    dynamic_signal_types: true
+owner:
+  github: elastic/elastic-package
+  type: elastic

internal/testrunner/runners/system/tester.go

@@ -1974,9 +1974,17 @@ func (r *tester) expectedDatasets(scenario *scenarioTest, config *testConfig) ([
 		// get dataset directly from package policy added when preparing the scenario
 		expectedDataset := scenario.dataStreamDataset
 		if scenario.policyTemplate.Input == otelCollectorInputName {
-			// Input packages whose input is `otelcol` must add the `.otel` suffix
-			// Example: httpcheck.metrics.otel
-			expectedDataset += "." + otelSuffixDataset
+			if scenario.policyTemplate.DynamicSignalTypes {
+				// For dynamic_signal_types packages, Fleet stores "generic.otel" as the
+				// policy dataset, but the OTel collector routes documents to scope-based
+				// data streams (e.g. <policyTemplateName>.otel). Derive the expected
+				// dataset from the policy template name, not the policy variable.
+				expectedDataset = scenario.policyTemplate.Name + "." + otelSuffixDataset
+			} else {
+				// Input packages whose input is `otelcol` must add the `.otel` suffix
+				// Example: httpcheck.metrics.otel
+				expectedDataset += "." + otelSuffixDataset
+			}
 			// Traces can also emit to a shared logs data stream (e.g. logs-generic.otel-*).
 			expectedDatasets = []string{expectedDataset, "generic." + otelSuffixDataset}
 		} else {

test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.expected

@@ -0,0 +1,79 @@
+connectors:
+    forward: {}
+exporters:
+    elasticsearch/componentid-0:
+        endpoints:
+            - https://elasticsearch:9200
+inputs: []
+output_permissions:
+    default:
+        _elastic_agent_checks:
+            cluster:
+                - monitor
+        _elastic_agent_monitoring:
+            indices: []
+        uuid-for-permissions-on-related-indices:
+            indices:
+                - names:
+                    - logs-*-*
+                  privileges:
+                    - auto_configure
+                    - create_doc
+                - names:
+                    - metrics-*-*
+                  privileges:
+                    - auto_configure
+                    - create_doc
+processors:
+    resourcedetection/componentid-0:
+        detectors:
+            - system
+    transform/componentid-1:
+        log_statements:
+            - context: log
+              statements:
+                - set(attributes["data_stream.type"], "logs")
+                - set(attributes["data_stream.namespace"], "ep")
+        metric_statements:
+            - context: datapoint
+              statements:
+                - set(attributes["data_stream.type"], "metrics")
+                - set(attributes["data_stream.namespace"], "ep")
+receivers:
+    sqlserver/componentid-0:
+        collection_interval: 10s
+        events:
+            db.server.query_sample:
+                enabled: false
+            db.server.top_query:
+                enabled: false
+        initial_delay: 1s
+secret_references: []
+service:
+    pipelines:
+        logs:
+            exporters:
+                - elasticsearch/componentid-0
+            receivers:
+                - forward
+        logs/componentid-0:
+            exporters:
+                - forward
+            processors:
+                - resourcedetection/componentid-0
+                - transform/componentid-1
+            receivers:
+                - sqlserver/componentid-0
+        metrics:
+            exporters:
+                - elasticsearch/componentid-0
+            receivers:
+                - forward
+        metrics/componentid-1:
+            exporters:
+                - forward
+            processors:
+                - resourcedetection/componentid-0
+                - transform/componentid-1
+            receivers:
+                - sqlserver/componentid-0

test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.yml

@@ -0,0 +1 @@
+vars: ~

test/packages/parallel/sql_server_input_otel/sample_event.json

@@ -1,43 +1,42 @@
 {
-    "@timestamp": "2026-03-23T18:57:30.184Z",
-    "_metric_names_hash": "8986448896debcf6",
+    "@timestamp": "2026-04-07T11:03:39.789Z",
+    "_metric_names_hash": "7ee276998c966647",
+    "attributes": {
+        "performance_counter": {
+            "object_name": "SQLServer:Buffer Node"
+        }
+    },
     "data_stream": {
         "dataset": "sqlserverreceiver.otel",
-        "namespace": "25898",
+        "namespace": "23377",
         "type": "metrics"
     },
     "event": {
-        "agent_id_status": "missing",
-        "dataset": "sqlserverreceiver.otel",
-        "ingested": "2026-03-23T18:57:40Z"
+        "dataset": "sqlserverreceiver.otel"
     },
     "host": {
-        "name": "elastic-agent-58948",
+        "name": "elastic-agent-57639",
         "os": {
             "platform": "linux"
         }
     },
     "metrics": {
         "sqlserver": {
-            "batch": {
-                "request": {
-                    "rate": 64
-                }
-            },
-            "lock": {
-                "wait": {
-                    "rate": 3
-                }
+            "page": {
+                "life_expectancy": 41
             }
         }
     },
     "os": {
         "type": "linux"
     },
+    "performance_counter": {
+        "object_name": "SQLServer:Buffer Node"
+    },
     "resource": {
         "attributes": {
             "host": {
-                "name": "elastic-agent-58948"
+                "name": "elastic-agent-57639"
             },
             "os": {
                 "type": "linux"
@@ -48,7 +47,7 @@
                 }
             }
         },
-        "schema_url": "https://opentelemetry.io/schemas/1.38.0"
+        "schema_url": "https://opentelemetry.io/schemas/1.39.0"
     },
     "scope": {
         "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver",
@@ -63,17 +62,10 @@
         }
     },
     "sqlserver": {
-        "batch": {
-            "request": {
-                "rate": 64
-            }
-        },
-        "lock": {
-            "wait": {
-                "rate": 3
-            }
+        "page": {
+            "life_expectancy": 41
         }
     },
-    "start_timestamp": "2026-03-23T18:57:29.073Z",
-    "unit": "{requests}/s"
+    "start_timestamp": "2026-04-07T11:03:38.605Z",
+    "unit": "s"
 }