From 3693afb3097180f2388823c8f4ba12798900c393 Mon Sep 17 00:00:00 2001 From: Tere Date: Tue, 7 Apr 2026 12:28:40 +0200 Subject: [PATCH 01/12] fix: use generic.otel stream key for dynamic_signal_types input packages Fleet changed (April 2026) to use "generic.otel" as the stream path/dataset for input packages with dynamic_signal_types: true, instead of the previous . default. Align BuildInputPackagePolicy to send the matching stream key and data_stream.dataset variable in simplified API requests. The test validator's expectedDatasets() was deriving expected document datasets from the policy variable (now "generic.otel"), but indexed documents retain the OTel scope-based dataset (e.g. sqlserverreceiver.otel). For dynamic_signal_types packages, derive the expected dataset from policyTemplate.Name + ".otel" instead. Co-Authored-By: Claude Sonnet 4.6 --- internal/kibana/packagepolicy.go | 8 ++++ internal/kibana/packagepolicy_test.go | 20 +++++++++ .../testdata/otel_input_dynamic_signals.json | 24 +++++++++++ .../otel_input_dynamic_signals_legacy.json | 42 +++++++++++++++++++ .../packages/otel_dynamic_input/manifest.yml | 23 ++++++++++ internal/testrunner/runners/system/tester.go | 14 +++++-- 6 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 internal/kibana/testdata/otel_input_dynamic_signals.json create mode 100644 internal/kibana/testdata/otel_input_dynamic_signals_legacy.json create mode 100644 internal/kibana/testdata/packages/otel_dynamic_input/manifest.yml diff --git a/internal/kibana/packagepolicy.go b/internal/kibana/packagepolicy.go index 3e2969cbee..420b0b3f8a 100644 --- a/internal/kibana/packagepolicy.go +++ b/internal/kibana/packagepolicy.go @@ -161,7 +161,12 @@ func BuildInputPackagePolicy( } // Dataset key for the stream: .. + // For packages with dynamic_signal_types, Fleet uses "generic.otel" as the + // stream path, so we must match that key in the simplified API request. streamDataset := fmt.Sprintf("%s.%s", manifest.Name, policyTemplate.Name) + if policyTemplate.DynamicSignalTypes { + streamDataset = "generic.otel" + } vars := SetKibanaVariables(policyTemplate.Vars, varValues) ensureDatasetVar(vars, policyTemplate, varValues) @@ -235,6 +240,9 @@ func ensureDatasetVar(vars Vars, policyTemplate packages.PolicyTemplate, varValu return } dataset := policyTemplate.Name + if policyTemplate.DynamicSignalTypes { + dataset = "generic.otel" + } for _, def := range policyTemplate.Vars { if def.Name == "data_stream.dataset" && def.Default != nil { if s, ok := def.Default.Value().(string); ok && s != "" { diff --git a/internal/kibana/packagepolicy_test.go b/internal/kibana/packagepolicy_test.go index 8519c0bb6c..fd2f6fe14f 100644 --- a/internal/kibana/packagepolicy_test.go +++ b/internal/kibana/packagepolicy_test.go @@ -282,6 +282,19 @@ func TestBuildInputPackagePolicy(t *testing.T) { goldenSimplified: "testdata/otel_traces_use_apm.json", goldenLegacy: "testdata/otel_traces_use_apm_legacy.json", }, + { + // Input package with dynamic_signal_types: true. Fleet now uses + // "generic.otel" as the stream path/dataset for such packages, so + // BuildInputPackagePolicy must use "generic.otel" as the stream key + // and the data_stream.dataset variable value. + name: "otel_input_dynamic_signals", + packageRoot: "testdata/packages/otel_dynamic_input", + policyTemplateName: "sqlreceiver", + policyName: "otel-dynamic-test", + varValues: common.MapStr{}, + goldenSimplified: "testdata/otel_input_dynamic_signals.json", + goldenLegacy: "testdata/otel_input_dynamic_signals_legacy.json", + }, { // Package-level variable: the user overrides the default package-level // var (custom_tag). BuildInputPackagePolicy must forward manifest.Vars @@ -515,6 +528,13 @@ func TestEnsureDatasetVar(t *testing.T) { varValues: common.MapStr{}, wantDataset: "sql_query", }, + { + name: "dynamic_signal_types uses generic.otel", + vars: Vars{}, + policyTemplate: packages.PolicyTemplate{Name: "sqlreceiver", DynamicSignalTypes: true}, + varValues: common.MapStr{}, + wantDataset: "generic.otel", + }, } for _, c := range cases { diff --git a/internal/kibana/testdata/otel_input_dynamic_signals.json b/internal/kibana/testdata/otel_input_dynamic_signals.json new file mode 100644 index 0000000000..be15b54229 --- /dev/null +++ b/internal/kibana/testdata/otel_input_dynamic_signals.json @@ -0,0 +1,24 @@ +{ + "name": "otel-dynamic-test", + "description": "", + "namespace": "test", + "policy_id": "test-policy-id", + "package": { + "name": "otel_dynamic_input", + "version": "0.1.0" + }, + "inputs": { + "sqlreceiver-otelcol": { + "enabled": true, + "streams": { + "generic.otel": { + "enabled": true, + "vars": { + "data_stream.dataset": "generic.otel" + } + } + } + } + }, + "force": false +} diff --git a/internal/kibana/testdata/otel_input_dynamic_signals_legacy.json b/internal/kibana/testdata/otel_input_dynamic_signals_legacy.json new file mode 100644 index 0000000000..70cf183578 --- /dev/null +++ b/internal/kibana/testdata/otel_input_dynamic_signals_legacy.json @@ -0,0 +1,42 @@ +{ + "name": "otel-dynamic-test", + "description": "", + "namespace": "test", + "policy_id": "test-policy-id", + "enabled": true, + "package": { + "name": "otel_dynamic_input", + "title": "OTel Dynamic Signals Input Package", + "version": "0.1.0" + }, + "inputs": [ + { + "policy_template": "sqlreceiver", + "type": "otelcol", + "enabled": true, + "vars": { + "data_stream.dataset": { + "value": "generic.otel", + "type": "text" + } + }, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "", + "dataset": "generic.otel" + }, + "vars": { + "data_stream.dataset": { + "value": "generic.otel", + "type": "text" + } + } + } + ] + } + ], + "output_id": "", + "force": false +} diff --git a/internal/kibana/testdata/packages/otel_dynamic_input/manifest.yml b/internal/kibana/testdata/packages/otel_dynamic_input/manifest.yml new file mode 100644 index 0000000000..a347eaead1 --- /dev/null +++ b/internal/kibana/testdata/packages/otel_dynamic_input/manifest.yml @@ -0,0 +1,23 @@ +format_version: 3.5.0 +name: otel_dynamic_input +title: OTel Dynamic Signals Input Package +description: >- + Minimal OTel input package with dynamic_signal_types used to verify that + BuildInputPackagePolicy uses "generic.otel" as the stream key. +type: input +version: 0.1.0 +categories: + - observability +conditions: + kibana: + version: "^9.4.0" +policy_templates: + - name: sqlreceiver + title: SQL Receiver + description: Collect SQL metrics and logs using OTel. + input: otelcol + template_path: input.yml.hbs + dynamic_signal_types: true +owner: + github: elastic/elastic-package + type: elastic diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 17ccf7db87..6b4642af88 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -1974,9 +1974,17 @@ func (r *tester) expectedDatasets(scenario *scenarioTest, config *testConfig) ([ // get dataset directly from package policy added when preparing the scenario expectedDataset := scenario.dataStreamDataset if scenario.policyTemplate.Input == otelCollectorInputName { - // Input packages whose input is `otelcol` must add the `.otel` suffix - // Example: httpcheck.metrics.otel - expectedDataset += "." + otelSuffixDataset + if scenario.policyTemplate.DynamicSignalTypes { + // For dynamic_signal_types packages, Fleet stores "generic.otel" as the + // policy dataset, but the OTel collector routes documents to scope-based + // data streams (e.g. .otel). Derive the expected + // dataset from the policy template name, not the policy variable. + expectedDataset = scenario.policyTemplate.Name + "." + otelSuffixDataset + } else { + // Input packages whose input is `otelcol` must add the `.otel` suffix + // Example: httpcheck.metrics.otel + expectedDataset += "." + otelSuffixDataset + } // Traces can also emit to a shared logs data stream (e.g. logs-generic.otel-*). expectedDatasets = []string{expectedDataset, "generic." + otelSuffixDataset} } else { From d1882d6d4f046eb3b74c282ac7d51ed8bcce253d Mon Sep 17 00:00:00 2001 From: Tere Date: Tue, 7 Apr 2026 13:14:56 +0200 Subject: [PATCH 02/12] test: add policy test fixtures for dynamic_signal_types and zipkin otel Add policy test config and expected output for sql_server_input_otel (dynamic_signal_types: true), generated against a live stack. Update zipkin_input_otel expected files to reflect Fleet no longer setting data_stream.dataset in OTel trace routing transforms. Add sample event files for both packages. Co-Authored-By: Claude Sonnet 4.6 --- .../test/policy/test-default-config.expected | 79 +++++++++++++ .../_dev/test/policy/test-default-config.yml | 1 + .../sql_server_input_otel/sample_event.json | 50 ++++---- .../sample_event_logs.json | 102 +++++++--------- .../sample_event_metrics.json | 50 ++++---- .../test-default-custom-dataset.expected | 1 - .../test-default-disable-use-apm.expected | 1 - .../test/policy/test-default-use-apm.expected | 1 - .../test/policy/test-default-vars.expected | 1 - .../_dev/test/policy/test-default.expected | 1 - .../zipkin_input_otel/sample_event_logs.json | 50 ++++++++ .../sample_event_traces.json | 111 ++++++++++++++++++ 12 files changed, 325 insertions(+), 123 deletions(-) create mode 100644 test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.expected create mode 100644 test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.yml create mode 100644 test/packages/parallel/zipkin_input_otel/sample_event_logs.json create mode 100644 test/packages/parallel/zipkin_input_otel/sample_event_traces.json diff --git a/test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.expected b/test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.expected new file mode 100644 index 0000000000..fc9d284b0f --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.expected @@ -0,0 +1,79 @@ +connectors: + forward: {} +exporters: + elasticsearch/componentid-0: + endpoints: + - https://elasticsearch:9200 +inputs: [] +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-*-* + privileges: + - auto_configure + - create_doc + - names: + - metrics-*-* + privileges: + - auto_configure + - create_doc +processors: + resourcedetection/componentid-0: + detectors: + - system + transform/componentid-1: + log_statements: + - context: log + statements: + - set(attributes["data_stream.type"], "logs") + - set(attributes["data_stream.namespace"], "ep") + metric_statements: + - context: datapoint + statements: + - set(attributes["data_stream.type"], "metrics") + - set(attributes["data_stream.namespace"], "ep") +receivers: + sqlserver/componentid-0: + collection_interval: 10s + events: + db.server.query_sample: + enabled: false + db.server.top_query: + enabled: false + initial_delay: 1s +secret_references: [] +service: + pipelines: + logs: + exporters: + - elasticsearch/componentid-0 + receivers: + - forward + logs/componentid-0: + exporters: + - forward + processors: + - resourcedetection/componentid-0 + - transform/componentid-1 + receivers: + - sqlserver/componentid-0 + metrics: + exporters: + - elasticsearch/componentid-0 + receivers: + - forward + metrics/componentid-1: + exporters: + - forward + processors: + - resourcedetection/componentid-0 + - transform/componentid-1 + receivers: + - sqlserver/componentid-0 diff --git a/test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.yml b/test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.yml new file mode 100644 index 0000000000..f057622ddd --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.yml @@ -0,0 +1 @@ +vars: ~ \ No newline at end of file diff --git a/test/packages/parallel/sql_server_input_otel/sample_event.json b/test/packages/parallel/sql_server_input_otel/sample_event.json index b0a6b84033..2e15c06e72 100644 --- a/test/packages/parallel/sql_server_input_otel/sample_event.json +++ b/test/packages/parallel/sql_server_input_otel/sample_event.json @@ -1,43 +1,42 @@ { - "@timestamp": "2026-03-23T18:57:30.184Z", - "_metric_names_hash": "8986448896debcf6", + "@timestamp": "2026-04-07T11:03:39.789Z", + "_metric_names_hash": "7ee276998c966647", + "attributes": { + "performance_counter": { + "object_name": "SQLServer:Buffer Node" + } + }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "25898", + "namespace": "23377", "type": "metrics" }, "event": { - "agent_id_status": "missing", - "dataset": "sqlserverreceiver.otel", - "ingested": "2026-03-23T18:57:40Z" + "dataset": "sqlserverreceiver.otel" }, "host": { - "name": "elastic-agent-58948", + "name": "elastic-agent-57639", "os": { "platform": "linux" } }, "metrics": { "sqlserver": { - "batch": { - "request": { - "rate": 64 - } - }, - "lock": { - "wait": { - "rate": 3 - } + "page": { + "life_expectancy": 41 } } }, "os": { "type": "linux" }, + "performance_counter": { + "object_name": "SQLServer:Buffer Node" + }, "resource": { "attributes": { "host": { - "name": "elastic-agent-58948" + "name": "elastic-agent-57639" }, "os": { "type": "linux" @@ -48,7 +47,7 @@ } } }, - "schema_url": "https://opentelemetry.io/schemas/1.38.0" + "schema_url": "https://opentelemetry.io/schemas/1.39.0" }, "scope": { "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver", @@ -63,17 +62,10 @@ } }, "sqlserver": { - "batch": { - "request": { - "rate": 64 - } - }, - "lock": { - "wait": { - "rate": 3 - } + "page": { + "life_expectancy": 41 } }, - "start_timestamp": "2026-03-23T18:57:29.073Z", - "unit": "{requests}/s" + "start_timestamp": "2026-04-07T11:03:38.605Z", + "unit": "s" } diff --git a/test/packages/parallel/sql_server_input_otel/sample_event_logs.json b/test/packages/parallel/sql_server_input_otel/sample_event_logs.json index 0b97fb0991..6699b0b2c8 100644 --- a/test/packages/parallel/sql_server_input_otel/sample_event_logs.json +++ b/test/packages/parallel/sql_server_input_otel/sample_event_logs.json @@ -1,90 +1,84 @@ { - "@timestamp": "2026-03-23T18:56:05.895Z", + "@timestamp": "2026-04-07T11:01:48.355Z", "attributes": { "client": { - "address": "elastic-agent-75417", - "port": 36936 + "address": "elastic-agent-10224", + "port": 48948 }, "db": { "namespace": "master", - "query": { - "text": "WITH PerfCounters SELECT DISTINCT RTRIM ( spi. object_name ) object_name, RTRIM ( spi. counter_name ) counter_name, RTRIM ( spi. instance_name ), CAST ( spi. cntr_value ), spi. cntr_type FROM sys.dm_os_performance_counters WHERE counter_name IN ( ? ) OR ( spi. object_name LIKE ? OR spi. object_name LIKE ? OR spi. object_name LIKE ? ) OR ( spi. instance_name IN ( ? ) AND spi. counter_name IN ( ? ) ) ) INSERT INTO @PCounters SELECT * FROM PerfCounters" - }, "system": { "name": "microsoft.sql_server" } }, "network": { "peer": { - "address": "172.20.0.2", - "port": 36936 + "address": "172.29.0.2", + "port": 48948 } }, "sqlserver": { "blocking_session_id": 0, - "command": "INSERT", + "command": "SELECT", "context_info": "", - "cpu_time": 0.043, - "deadlock_priority": -10, + "cpu_time": 0.046, + "deadlock_priority": 0, "estimated_completion_time": 0, "lock_timeout": -0.001, - "logical_reads": 607, + "logical_reads": 669, "open_transaction_count": 0, "percent_complete": 0, - "query_hash": "6df033ef55623a4a", - "query_plan_hash": "49c16c014d7e4505", - "query_start": "2026-03-23T18:56:05.837+00:00", - "reads": 40, + "procedure_id": "0", + "procedure_name": "", + "query_hash": "30447d7d5ef3066a", + "query_plan_hash": "0c97dc34ff850504", + "query_start": "2026-04-07T11:01:48.297+00:00", + "reads": 104, "request_status": "running", - "row_count": 1, + "row_count": 0, "session_id": 51, "session_status": "running", - "total_elapsed_time": 0.043, - "transaction_id": 0, + "total_elapsed_time": 0.047, + "transaction_id": 1899, "transaction_isolation_level": 2, "wait_resource": "", "wait_time": 0, "wait_type": "", - "writes": 19 + "writes": 0 }, "user": { "name": "sa" } }, "client": { - "address": "elastic-agent-75417", - "port": 36936 + "address": "elastic-agent-10224", + "port": 48948 }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "73861", + "namespace": "19985", "type": "logs" }, "db": { "namespace": "master", - "query": { - "text": "WITH PerfCounters SELECT DISTINCT RTRIM ( spi. object_name ) object_name, RTRIM ( spi. counter_name ) counter_name, RTRIM ( spi. instance_name ), CAST ( spi. cntr_value ), spi. cntr_type FROM sys.dm_os_performance_counters WHERE counter_name IN ( ? ) OR ( spi. object_name LIKE ? OR spi. object_name LIKE ? OR spi. object_name LIKE ? ) OR ( spi. instance_name IN ( ? ) AND spi. counter_name IN ( ? ) ) ) INSERT INTO @PCounters SELECT * FROM PerfCounters" - }, "system": { "name": "microsoft.sql_server" } }, "event": { - "agent_id_status": "missing", - "dataset": "sqlserverreceiver.otel", - "ingested": "2026-03-23T18:56:15Z" + "dataset": "sqlserverreceiver.otel" }, "event_name": "db.server.query_sample", "host": { - "name": "elastic-agent-75417", + "name": "elastic-agent-10224", "os": { "platform": "linux" } }, "network": { "peer": { - "address": "172.20.0.2", - "port": 36936 + "address": "172.29.0.2", + "port": 48948 } }, "observed_timestamp": "1970-01-01T00:00:00.000Z", @@ -94,7 +88,7 @@ "resource": { "attributes": { "host": { - "name": "elastic-agent-75417" + "name": "elastic-agent-10224" }, "os": { "type": "linux" @@ -103,17 +97,9 @@ "instance": { "id": "svc-sql_server_input_otel:1433" } - }, - "sqlserver": { - "computer": { - "name": "elastic-agent-75417" - }, - "instance": { - "name": "3c03efd12c01" - } } }, - "schema_url": "https://opentelemetry.io/schemas/1.38.0" + "schema_url": "https://opentelemetry.io/schemas/1.39.0" }, "scope": { "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver", @@ -129,36 +115,32 @@ }, "sqlserver": { "blocking_session_id": 0, - "command": "INSERT", - "computer": { - "name": "elastic-agent-75417" - }, + "command": "SELECT", "context_info": "", - "cpu_time": 0.043, - "deadlock_priority": -10, + "cpu_time": 0.046, + "deadlock_priority": 0, "estimated_completion_time": 0, - "instance": { - "name": "3c03efd12c01" - }, "lock_timeout": -0.001, - "logical_reads": 607, + "logical_reads": 669, "open_transaction_count": 0, "percent_complete": 0, - "query_hash": "6df033ef55623a4a", - "query_plan_hash": "49c16c014d7e4505", - "query_start": "2026-03-23T18:56:05.837+00:00", - "reads": 40, + "procedure_id": "0", + "procedure_name": "", + "query_hash": "30447d7d5ef3066a", + "query_plan_hash": "0c97dc34ff850504", + "query_start": "2026-04-07T11:01:48.297+00:00", + "reads": 104, "request_status": "running", - "row_count": 1, + "row_count": 0, "session_id": 51, "session_status": "running", - "total_elapsed_time": 0.043, - "transaction_id": 0, + "total_elapsed_time": 0.047, + "transaction_id": 1899, "transaction_isolation_level": 2, "wait_resource": "", "wait_time": 0, "wait_type": "", - "writes": 19 + "writes": 0 }, "user": { "name": "sa" diff --git a/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json b/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json index bb204294f6..f132f8a3a0 100644 --- a/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json +++ b/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json @@ -1,43 +1,42 @@ { - "@timestamp": "2026-03-23T18:56:05.948Z", - "_metric_names_hash": "8986448896debcf6", + "@timestamp": "2026-04-07T11:01:48.409Z", + "_metric_names_hash": "7ee276998c966647", + "attributes": { + "performance_counter": { + "object_name": "SQLServer:Buffer Node" + } + }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "73861", + "namespace": "19985", "type": "metrics" }, "event": { - "agent_id_status": "missing", - "dataset": "sqlserverreceiver.otel", - "ingested": "2026-03-23T18:56:15Z" + "dataset": "sqlserverreceiver.otel" }, "host": { - "name": "elastic-agent-75417", + "name": "elastic-agent-10224", "os": { "platform": "linux" } }, "metrics": { "sqlserver": { - "batch": { - "request": { - "rate": 65 - } - }, - "lock": { - "wait": { - "rate": 5 - } + "page": { + "life_expectancy": 22 } } }, "os": { "type": "linux" }, + "performance_counter": { + "object_name": "SQLServer:Buffer Node" + }, "resource": { "attributes": { "host": { - "name": "elastic-agent-75417" + "name": "elastic-agent-10224" }, "os": { "type": "linux" @@ -48,7 +47,7 @@ } } }, - "schema_url": "https://opentelemetry.io/schemas/1.38.0" + "schema_url": "https://opentelemetry.io/schemas/1.39.0" }, "scope": { "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver", @@ -63,17 +62,10 @@ } }, "sqlserver": { - "batch": { - "request": { - "rate": 65 - } - }, - "lock": { - "wait": { - "rate": 5 - } + "page": { + "life_expectancy": 22 } }, - "start_timestamp": "2026-03-23T18:56:04.825Z", - "unit": "{requests}/s" + "start_timestamp": "2026-04-07T11:01:47.281Z", + "unit": "s" } diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected index cddb652cb4..735a92b5d0 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected @@ -42,7 +42,6 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") - - set(attributes["data_stream.dataset"], "zipkin.custom") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected index 9bfab820c3..3f6b7eb6fc 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected @@ -30,7 +30,6 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") - - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected index 005a7c72b6..735a92b5d0 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected @@ -42,7 +42,6 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") - - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected index c23cff1583..ab89a91b4c 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected @@ -42,7 +42,6 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") - - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected index 005a7c72b6..735a92b5d0 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected @@ -42,7 +42,6 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") - - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/sample_event_logs.json b/test/packages/parallel/zipkin_input_otel/sample_event_logs.json new file mode 100644 index 0000000000..8549006d1f --- /dev/null +++ b/test/packages/parallel/zipkin_input_otel/sample_event_logs.json @@ -0,0 +1,50 @@ +{ + "@timestamp": "2026-04-07T10:57:35.581Z", + "agent": { + "name": "otlp", + "version": "unknown" + }, + "attributes": { + "event": { + "name": "wr" + }, + "timestamp": { + "us": 1775559455581041 + } + }, + "data_stream": { + "dataset": "generic.otel", + "namespace": "88673", + "type": "logs" + }, + "event": { + "dataset": "generic.otel", + "name": "wr" + }, + "event_name": "wr", + "resource": { + "attributes": { + "agent": { + "name": "otlp", + "version": "unknown" + }, + "service": { + "name": "backend" + } + } + }, + "service": { + "name": "backend" + }, + "span": { + "id": "e86a426e61162cce" + }, + "span_id": "e86a426e61162cce", + "timestamp": { + "us": 1775559455581041 + }, + "trace": { + "id": "0000000000000000e86a426e61162cce" + }, + "trace_id": "0000000000000000e86a426e61162cce" +} diff --git a/test/packages/parallel/zipkin_input_otel/sample_event_traces.json b/test/packages/parallel/zipkin_input_otel/sample_event_traces.json new file mode 100644 index 0000000000..2011a36fff --- /dev/null +++ b/test/packages/parallel/zipkin_input_otel/sample_event_traces.json @@ -0,0 +1,111 @@ +{ + "@timestamp": "2026-04-07T10:57:35.580Z", + "agent": { + "name": "otlp", + "version": "unknown" + }, + "attributes": { + "event": { + "outcome": "success", + "success_count": 1 + }, + "http": { + "method": "GET", + "path": "/api" + }, + "net": { + "host": { + "ip": "172.30.0.2" + }, + "peer": { + "ip": "172.30.0.3", + "port": 45894 + } + }, + "processor": { + "event": "transaction" + }, + "timestamp": { + "us": 1775559455580995 + }, + "transaction": { + "duration": { + "us": 709 + }, + "id": "e86a426e61162cce", + "name": "GET /api", + "representative_count": 1, + "result": "Success", + "root": true, + "sampled": true, + "type": "request" + } + }, + "data_stream": { + "dataset": "generic.otel", + "namespace": "88673", + "type": "traces" + }, + "duration": 709000, + "event": { + "dataset": "generic.otel", + "outcome": "success", + "success_count": 1 + }, + "http": { + "method": "GET", + "path": "/api" + }, + "kind": "Server", + "name": "GET /api", + "net": { + "host": { + "ip": "172.30.0.2" + }, + "peer": { + "ip": "172.30.0.3", + "port": 45894 + } + }, + "processor": { + "event": "transaction" + }, + "resource": { + "attributes": { + "agent": { + "name": "otlp", + "version": "unknown" + }, + "service": { + "name": "backend" + } + } + }, + "service": { + "name": "backend" + }, + "span": { + "id": "e86a426e61162cce", + "name": "GET /api" + }, + "span_id": "e86a426e61162cce", + "timestamp": { + "us": 1775559455580995 + }, + "trace": { + "id": "0000000000000000e86a426e61162cce" + }, + "trace_id": "0000000000000000e86a426e61162cce", + "transaction": { + "duration": { + "us": 709 + }, + "id": "e86a426e61162cce", + "name": "GET /api", + "representative_count": 1, + "result": "Success", + "root": true, + "sampled": true, + "type": "request" + } +} From a6e6349436d2e2c5566ad344dc7923f8c4ad8add Mon Sep 17 00:00:00 2001 From: Tere Date: Tue, 7 Apr 2026 13:57:47 +0200 Subject: [PATCH 03/12] fix: update example event in README for sql_server_input_otel --- .../sql_server_input_otel/docs/README.md | 50 ++++++++----------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/test/packages/parallel/sql_server_input_otel/docs/README.md b/test/packages/parallel/sql_server_input_otel/docs/README.md index 2b255b7674..61c6281272 100644 --- a/test/packages/parallel/sql_server_input_otel/docs/README.md +++ b/test/packages/parallel/sql_server_input_otel/docs/README.md @@ -57,45 +57,44 @@ An example event looks as following: ```json { - "@timestamp": "2026-03-23T18:57:30.184Z", - "_metric_names_hash": "8986448896debcf6", + "@timestamp": "2026-04-07T11:03:39.789Z", + "_metric_names_hash": "7ee276998c966647", + "attributes": { + "performance_counter": { + "object_name": "SQLServer:Buffer Node" + } + }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "25898", + "namespace": "23377", "type": "metrics" }, "event": { - "agent_id_status": "missing", - "dataset": "sqlserverreceiver.otel", - "ingested": "2026-03-23T18:57:40Z" + "dataset": "sqlserverreceiver.otel" }, "host": { - "name": "elastic-agent-58948", + "name": "elastic-agent-57639", "os": { "platform": "linux" } }, "metrics": { "sqlserver": { - "batch": { - "request": { - "rate": 64 - } - }, - "lock": { - "wait": { - "rate": 3 - } + "page": { + "life_expectancy": 41 } } }, "os": { "type": "linux" }, + "performance_counter": { + "object_name": "SQLServer:Buffer Node" + }, "resource": { "attributes": { "host": { - "name": "elastic-agent-58948" + "name": "elastic-agent-57639" }, "os": { "type": "linux" @@ -106,7 +105,7 @@ An example event looks as following: } } }, - "schema_url": "https://opentelemetry.io/schemas/1.38.0" + "schema_url": "https://opentelemetry.io/schemas/1.39.0" }, "scope": { "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver", @@ -121,18 +120,11 @@ An example event looks as following: } }, "sqlserver": { - "batch": { - "request": { - "rate": 64 - } - }, - "lock": { - "wait": { - "rate": 3 - } + "page": { + "life_expectancy": 41 } }, - "start_timestamp": "2026-03-23T18:57:29.073Z", - "unit": "{requests}/s" + "start_timestamp": "2026-04-07T11:03:38.605Z", + "unit": "s" } ``` From 0a9330a3b6d97c3dff194633247d94c66453c7ad Mon Sep 17 00:00:00 2001 From: Tere Date: Tue, 7 Apr 2026 15:31:03 +0200 Subject: [PATCH 04/12] add mysql input otel package to test fixtures with dataset override --- .../parallel/mysql_input_otel/LICENSE.txt | 93 +++++++ .../_dev/build/docs/README.md | 44 +++ .../_dev/deploy/docker/Dockerfile | 5 + .../_dev/deploy/docker/docker-compose.yml | 20 ++ .../_dev/deploy/docker/entrypoint.sh | 14 + .../mysql_input_otel/_dev/deploy/variants.yml | 4 + .../_dev/test/policy/test-default.expected | 108 +++++++ .../_dev/test/policy/test-default.yml | 2 + .../_dev/test/system/test-default-config.yml | 7 + .../agent/input/input.yml.hbs | 66 +++++ .../parallel/mysql_input_otel/changelog.yml | 20 ++ .../parallel/mysql_input_otel/docs/README.md | 44 +++ .../mysql_input_otel/img/logo_mysql_otel.svg | 16 ++ .../parallel/mysql_input_otel/manifest.yml | 263 ++++++++++++++++++ .../mysql_input_otel/sample_event_logs.json | 121 ++++++++ .../sample_event_metrics.json | 58 ++++ 16 files changed, 885 insertions(+) create mode 100644 test/packages/parallel/mysql_input_otel/LICENSE.txt create mode 100644 test/packages/parallel/mysql_input_otel/_dev/build/docs/README.md create mode 100644 test/packages/parallel/mysql_input_otel/_dev/deploy/docker/Dockerfile create mode 100644 test/packages/parallel/mysql_input_otel/_dev/deploy/docker/docker-compose.yml create mode 100644 test/packages/parallel/mysql_input_otel/_dev/deploy/docker/entrypoint.sh create mode 100644 test/packages/parallel/mysql_input_otel/_dev/deploy/variants.yml create mode 100644 test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected create mode 100644 test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.yml create mode 100644 test/packages/parallel/mysql_input_otel/_dev/test/system/test-default-config.yml create mode 100644 test/packages/parallel/mysql_input_otel/agent/input/input.yml.hbs create mode 100644 test/packages/parallel/mysql_input_otel/changelog.yml create mode 100644 test/packages/parallel/mysql_input_otel/docs/README.md create mode 100644 test/packages/parallel/mysql_input_otel/img/logo_mysql_otel.svg create mode 100644 test/packages/parallel/mysql_input_otel/manifest.yml create mode 100644 test/packages/parallel/mysql_input_otel/sample_event_logs.json create mode 100644 test/packages/parallel/mysql_input_otel/sample_event_metrics.json diff --git a/test/packages/parallel/mysql_input_otel/LICENSE.txt b/test/packages/parallel/mysql_input_otel/LICENSE.txt new file mode 100644 index 0000000000..809108b857 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/test/packages/parallel/mysql_input_otel/_dev/build/docs/README.md b/test/packages/parallel/mysql_input_otel/_dev/build/docs/README.md new file mode 100644 index 0000000000..edfd62e60d --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/_dev/build/docs/README.md @@ -0,0 +1,44 @@ +# MySQL OpenTelemetry Input Package + +## Overview + +The MySQL OpenTelemetry Input Package for Elastic enables collection of metrics and logs from MySQL database servers through OpenTelemetry protocols using the [mysqlreceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/mysqlreceiver). + +### How it works + +This package receives telemetry data from MySQL servers by configuring the MySQL endpoint and credentials in the Input Package, which then gets applied to the mysqlreceiver present in the EDOT collector, which then forwards the data to Elastic Agent. The Elastic Agent processes and enriches the data before sending it to Elasticsearch for indexing and analysis. Once the data arrives into Elasticsearch, its corresponding [MySQL OpenTelemetry Assets Package](https://www.elastic.co/docs/reference/integrations/mysql_otel) gets auto installed and the dashboards light up. + +The package collects both **metrics** (from `SHOW GLOBAL STATUS` and InnoDB tables) and **logs** (query samples and top queries). Log events are controlled by the Enable Query Sample Events and Enable Top Query Events settings: +- **`db.server.query_sample`**: Current running database statements (enabled by default) +- **`db.server.top_query`**: Queries that consumed the most CPU (disabled by default) + +## Requirements + +- MySQL 8.0+ or MariaDB 10.11+ +- A MySQL user with permissions to run `SHOW GLOBAL STATUS` +- For query sample collection, the `performance_schema` must be enabled + +### MySQL requirements for log collection + +To collect log events (query samples and top queries), configure MySQL as follows. For additional details, see the [upstream mysqlreceiver logs documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/mysqlreceiver#logs). + +| Parameter | Value | Description | +|-----------|-------|-------------| +| `performance_schema` | Enabled | Required for log collection | +| `max_digest_length` | 4096 | Recommended maximum length of digest text | +| `performance_schema_max_digest_length` | 4096 | Recommended maximum length of digest text on performance schema | +| `performance_schema_max_sql_text_length` | 4096 | Recommended maximum length of SQL text | + +Also grant the MySQL user access to the performance schema: + +```sql +GRANT SELECT ON performance_schema.* TO @'%'; +``` + +## Configuration + +For the full list of settings exposed for the receiver and examples, refer to the [configuration](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/mysqlreceiver#configuration) section. + +## Metrics and logs reference + +For a complete list of all available metrics, log events, and their detailed descriptions, refer to the [MySQL Receiver documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/mysqlreceiver/documentation.md) in the upstream OpenTelemetry Collector repository. diff --git a/test/packages/parallel/mysql_input_otel/_dev/deploy/docker/Dockerfile b/test/packages/parallel/mysql_input_otel/_dev/deploy/docker/Dockerfile new file mode 100644 index 0000000000..cee2deb382 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/_dev/deploy/docker/Dockerfile @@ -0,0 +1,5 @@ +ARG SERVICE_VERSION=8.4.8 +FROM mysql:${SERVICE_VERSION} +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/test/packages/parallel/mysql_input_otel/_dev/deploy/docker/docker-compose.yml b/test/packages/parallel/mysql_input_otel/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 0000000000..2b2b558d58 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,20 @@ +services: + mysql_input_otel: + build: + context: . + args: + SERVICE_VERSION: ${SERVICE_VERSION:-8.4.8} + ports: + - 3306 + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root"] + interval: 10s + retries: 5 + timeout: 5s + mysql_input_otel_is_ready: + image: tianon/true:multiarch + depends_on: + mysql_input_otel: + condition: service_healthy diff --git a/test/packages/parallel/mysql_input_otel/_dev/deploy/docker/entrypoint.sh b/test/packages/parallel/mysql_input_otel/_dev/deploy/docker/entrypoint.sh new file mode 100644 index 0000000000..a0ae816c53 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/_dev/deploy/docker/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -e +/usr/local/bin/docker-entrypoint.sh mysqld & +until mysql -u root -e "SELECT 1" 2>/dev/null; do sleep 2; done +mysql -u root -e "GRANT SELECT ON performance_schema.* TO 'root'@'%'; FLUSH PRIVILEGES;" 2>/dev/null || true +WORKLOAD_STARTED=0 +trap 'WORKLOAD_STARTED=1' HUP +echo "Waiting for SIGHUP to start workload..." +while [ "$WORKLOAD_STARTED" -eq 0 ]; do sleep 1; done +echo "SIGHUP received, starting workload..." +while true; do + mysql -u root -e "SELECT 1; SELECT SLEEP(2); SELECT COUNT(*) FROM information_schema.tables;" + sleep 1 +done diff --git a/test/packages/parallel/mysql_input_otel/_dev/deploy/variants.yml b/test/packages/parallel/mysql_input_otel/_dev/deploy/variants.yml new file mode 100644 index 0000000000..39f237d651 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/_dev/deploy/variants.yml @@ -0,0 +1,4 @@ +variants: + v8: + SERVICE_VERSION: "8.4.8" +default: v8 diff --git a/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected b/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected new file mode 100644 index 0000000000..3a727a0b89 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected @@ -0,0 +1,108 @@ +connectors: + forward: {} +exporters: + elasticsearch/componentid-0: + endpoints: + - https://elasticsearch:9200 +inputs: [] +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-*-* + privileges: + - auto_configure + - create_doc + - names: + - metrics-*-* + privileges: + - auto_configure + - create_doc +processors: + resourcedetection/componentid-0: + detectors: + - system + transform/componentid-1: + log_statements: + - context: log + statements: + - set(attributes["data_stream.type"], "logs") + - set(attributes["data_stream.namespace"], "ep") + metric_statements: + - context: datapoint + statements: + - set(attributes["data_stream.type"], "metrics") + - set(attributes["data_stream.namespace"], "ep") +receivers: + mysql/componentid-0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: false + initial_delay: 1s + metrics: + mysql.client.network.io: + enabled: true + mysql.commands: + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + enabled: true + mysql.max_used_connections: + enabled: true + mysql.query.client.count: + enabled: true + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.table_open_cache: + enabled: true + query_sample_collection: + max_rows_per_query: 100 + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h + transport: tcp + username: root +secret_references: [] +service: + pipelines: + logs: + exporters: + - elasticsearch/componentid-0 + receivers: + - forward + logs/componentid-0: + exporters: + - forward + processors: + - resourcedetection/componentid-0 + - transform/componentid-1 + receivers: + - mysql/componentid-0 + metrics: + exporters: + - elasticsearch/componentid-0 + receivers: + - forward + metrics/componentid-1: + exporters: + - forward + processors: + - resourcedetection/componentid-0 + - transform/componentid-1 + receivers: + - mysql/componentid-0 diff --git a/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.yml b/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.yml new file mode 100644 index 0000000000..12afef89c9 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.yml @@ -0,0 +1,2 @@ +vars: + data_stream.dataset: mysql_input_otel.test diff --git a/test/packages/parallel/mysql_input_otel/_dev/test/system/test-default-config.yml b/test/packages/parallel/mysql_input_otel/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..f76e74db2d --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +service: mysql_input_otel +service_notify_signal: SIGHUP +vars: + endpoint: "{{Hostname}}:{{Port}}" + username: "root" +assert: + min_count: 10 diff --git a/test/packages/parallel/mysql_input_otel/agent/input/input.yml.hbs b/test/packages/parallel/mysql_input_otel/agent/input/input.yml.hbs new file mode 100644 index 0000000000..bfa0fe8ca2 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/agent/input/input.yml.hbs @@ -0,0 +1,66 @@ +receivers: + mysql: + endpoint: {{endpoint}} + username: {{username}} +{{#if password}} + password: {{password}} +{{/if}} +{{#if database}} + database: {{database}} +{{/if}} + transport: {{transport}} + allow_native_passwords: {{allow_native_passwords}} +{{#if tls_enabled}} + tls: + insecure_skip_verify: {{tls_insecure_skip_verify}} +{{#if tls_ca_file}} + ca_file: {{tls_ca_file}} +{{/if}} +{{#if tls_cert_file}} + cert_file: {{tls_cert_file}} +{{/if}} +{{#if tls_key_file}} + key_file: {{tls_key_file}} +{{/if}} +{{#if tls_server_name_override}} + server_name_override: {{tls_server_name_override}} +{{/if}} +{{/if}} + collection_interval: {{collection_interval}} + initial_delay: {{initial_delay}} + statement_events: + digest_text_limit: {{statement_events_digest_text_limit}} + time_limit: {{statement_events_time_limit}} + limit: {{statement_events_limit}} + events: + db.server.query_sample: + enabled: {{enable_query_sample_events}} + db.server.top_query: + enabled: {{enable_top_query_events}} +{{#if enable_query_sample_events}} + query_sample_collection: + max_rows_per_query: {{query_sample_max_rows_per_query}} +{{/if}} +{{#if enable_top_query_events}} + top_query_collection: + lookback_time: {{top_query_lookback_time}} + max_query_sample_count: {{top_query_max_query_sample_count}} + top_query_count: {{top_query_count}} + collection_interval: {{top_query_collection_interval}} + query_plan_cache_size: {{top_query_plan_cache_size}} + query_plan_cache_ttl: {{top_query_plan_cache_ttl}} +{{/if}} +{{#if metrics}} + metrics: {{metrics}} +{{/if}} +processors: + resourcedetection/system: + detectors: ["system"] +service: + pipelines: + logs: + receivers: [mysql] + processors: [resourcedetection/system] + metrics: + receivers: [mysql] + processors: [resourcedetection/system] diff --git a/test/packages/parallel/mysql_input_otel/changelog.yml b/test/packages/parallel/mysql_input_otel/changelog.yml new file mode 100644 index 0000000000..a114755e47 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/changelog.yml @@ -0,0 +1,20 @@ +- version: "0.2.1" + changes: + - description: Remove type for dynamic signal type manifest field + type: bugfix + link: https://github.com/elastic/integrations/pull/17850 +- version: "0.2.0" + changes: + - description: Add logs support via multisignal (query samples, top queries) + type: enhancement + link: https://github.com/elastic/integrations/issues/17441 +- version: "0.1.1" + changes: + - description: Add metrics configuration + type: enhancement + link: https://github.com/elastic/integrations/pull/17449 +- version: "0.1.0" + changes: + - description: Initial release of MySQL OpenTelemetry input package + type: enhancement + link: https://github.com/elastic/integrations/pull/17090 diff --git a/test/packages/parallel/mysql_input_otel/docs/README.md b/test/packages/parallel/mysql_input_otel/docs/README.md new file mode 100644 index 0000000000..edfd62e60d --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/docs/README.md @@ -0,0 +1,44 @@ +# MySQL OpenTelemetry Input Package + +## Overview + +The MySQL OpenTelemetry Input Package for Elastic enables collection of metrics and logs from MySQL database servers through OpenTelemetry protocols using the [mysqlreceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/mysqlreceiver). + +### How it works + +This package receives telemetry data from MySQL servers by configuring the MySQL endpoint and credentials in the Input Package, which then gets applied to the mysqlreceiver present in the EDOT collector, which then forwards the data to Elastic Agent. The Elastic Agent processes and enriches the data before sending it to Elasticsearch for indexing and analysis. Once the data arrives into Elasticsearch, its corresponding [MySQL OpenTelemetry Assets Package](https://www.elastic.co/docs/reference/integrations/mysql_otel) gets auto installed and the dashboards light up. + +The package collects both **metrics** (from `SHOW GLOBAL STATUS` and InnoDB tables) and **logs** (query samples and top queries). Log events are controlled by the Enable Query Sample Events and Enable Top Query Events settings: +- **`db.server.query_sample`**: Current running database statements (enabled by default) +- **`db.server.top_query`**: Queries that consumed the most CPU (disabled by default) + +## Requirements + +- MySQL 8.0+ or MariaDB 10.11+ +- A MySQL user with permissions to run `SHOW GLOBAL STATUS` +- For query sample collection, the `performance_schema` must be enabled + +### MySQL requirements for log collection + +To collect log events (query samples and top queries), configure MySQL as follows. For additional details, see the [upstream mysqlreceiver logs documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/mysqlreceiver#logs). + +| Parameter | Value | Description | +|-----------|-------|-------------| +| `performance_schema` | Enabled | Required for log collection | +| `max_digest_length` | 4096 | Recommended maximum length of digest text | +| `performance_schema_max_digest_length` | 4096 | Recommended maximum length of digest text on performance schema | +| `performance_schema_max_sql_text_length` | 4096 | Recommended maximum length of SQL text | + +Also grant the MySQL user access to the performance schema: + +```sql +GRANT SELECT ON performance_schema.* TO @'%'; +``` + +## Configuration + +For the full list of settings exposed for the receiver and examples, refer to the [configuration](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/mysqlreceiver#configuration) section. + +## Metrics and logs reference + +For a complete list of all available metrics, log events, and their detailed descriptions, refer to the [MySQL Receiver documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/mysqlreceiver/documentation.md) in the upstream OpenTelemetry Collector repository. diff --git a/test/packages/parallel/mysql_input_otel/img/logo_mysql_otel.svg b/test/packages/parallel/mysql_input_otel/img/logo_mysql_otel.svg new file mode 100644 index 0000000000..198dfc7059 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/img/logo_mysql_otel.svg @@ -0,0 +1,16 @@ + + + Layer 1 + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/packages/parallel/mysql_input_otel/manifest.yml b/test/packages/parallel/mysql_input_otel/manifest.yml new file mode 100644 index 0000000000..6f082ba24b --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/manifest.yml @@ -0,0 +1,263 @@ +format_version: 3.6.0 +name: mysql_input_otel +title: "MySQL OpenTelemetry Input Package" +version: 0.2.1 +source: + license: "Elastic-2.0" +description: "Collect MySQL metrics and logs using OpenTelemetry Collector" +type: input +categories: + - datastore + - observability + - opentelemetry +conditions: + kibana: + version: "^9.4.0" + elastic: + subscription: "basic" +icons: + - src: /img/logo_mysql_otel.svg + title: MySQL logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: mysqlreceiver + title: MySQL Metrics and Logs (OpenTelemetry) + description: Collect MySQL metrics and logs (query samples, top queries) using OpenTelemetry Collector + input: otelcol + template_path: input.yml.hbs + dynamic_signal_types: true + vars: + # Connection settings + - name: endpoint + type: text + required: true + title: Endpoint + description: The endpoint of the MySQL server (host:port). + default: localhost:3306 + show_user: true + - name: username + type: text + required: true + title: Username + description: The username used to connect to MySQL. + default: root + show_user: true + - name: password + type: password + required: false + title: Password + description: The password used to connect to MySQL. + # TODO: Change to secret: true once Fleet Server resolves the secret handling issue + # for OTel input packages. See: https://github.com/elastic/fleet-server/issues/6277 + secret: false + show_user: true + - name: database + type: text + required: false + title: Database + description: The database name. If not specified, metrics will be collected for all databases. + show_user: true + - name: transport + type: select + required: false + title: Transport + description: Defines the network to use for connecting to the server. + multi: false + default: tcp + options: + - text: TCP + value: tcp + - text: TCP IPv4-only + value: tcp4 + - text: TCP IPv6-only + value: tcp6 + - text: Unix Socket + value: unix + show_user: false + - name: allow_native_passwords + type: bool + required: false + title: Allow Native Passwords + description: Allow the use of the native password authentication method. + default: true + secret: false + show_user: false + - name: tls_enabled + type: bool + required: false + title: Enable TLS Configuration + description: Enable TLS configuration for connecting to the server. + default: false + show_user: true + # TLS settings (only applies when TLS is enabled) + - name: tls_insecure_skip_verify + type: bool + required: false + title: Skip TLS Verification + description: Set to true to enable TLS but not verify the certificate. + default: false + show_user: false + - name: tls_server_name_override + type: text + required: false + title: TLS Server Name Override + description: Sets the ServerName in the TLSConfig. + show_user: false + - name: tls_ca_file + type: text + required: false + title: TLS CA File + description: Path to CA certificate file for verifying the server certificate. + show_user: false + - name: tls_cert_file + type: text + required: false + title: TLS Certificate File + description: Path to client certificate file for mutual TLS (mTLS) authentication. + show_user: false + - name: tls_key_file + type: text + required: false + title: TLS Key File + description: Path to client key file for mutual TLS (mTLS) authentication. + show_user: false + # Collection settings + - name: collection_interval + type: duration + required: false + title: Collection Interval + description: Time between each collection (e.g., 10s, 1m). + default: 10s + show_user: false + - name: initial_delay + type: duration + required: false + title: Initial Delay + description: Defines how long this receiver waits before starting. + default: 1s + show_user: false + # Statement events settings + - name: statement_events_digest_text_limit + type: integer + required: false + title: Statement Events Digest Text Limit + description: Maximum length of digest_text for statement events. + default: 120 + show_user: false + - name: statement_events_time_limit + type: duration + required: false + title: Statement Events Time Limit + description: Maximum time since the statements have been observed last time. + default: 24h + show_user: false + - name: statement_events_limit + type: integer + required: false + title: Statement Events Limit + description: Maximum number of statement event records (generated metrics). + default: 250 + show_user: false + # Log events (both require performance_schema and GRANT SELECT ON performance_schema.*) + # Defaults align with upstream mysqlreceiver: query_sample=true, top_query=false + # See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/mysqlreceiver/metadata.yaml#L236 + - name: enable_query_sample_events + type: bool + required: false + title: Enable Query Sample Events + description: Capture currently executing queries as log events (db.server.query_sample). + default: true + show_user: true + - name: enable_top_query_events + type: bool + required: false + title: Enable Top Query Events + description: Capture the most expensive queries as log events (db.server.top_query). + default: false + show_user: true + # Query sample collection settings (only applicable if enable_query_sample_events is true) + - name: query_sample_max_rows_per_query + type: integer + required: false + title: Query Sample Max Rows Per Query + description: Maximum number of rows to collect per scrape for query samples. + default: 100 + show_user: false + # Top query collection settings (only applicable if enable_top_query_events is true) + - name: top_query_lookback_time + type: integer + required: false + title: Top Query Lookback Time (seconds) + description: The time window in seconds to query for top queries. + default: 60 + show_user: false + - name: top_query_max_query_sample_count + type: integer + required: false + title: Top Query Max Sample Count + description: Maximum number of records to fetch in a single run. + default: 1000 + show_user: false + - name: top_query_count + type: integer + required: false + title: Top Query Count + description: Maximum number of active queries to report in a single run. + default: 200 + show_user: false + - name: top_query_collection_interval + type: duration + required: false + title: Top Query Collection Interval + description: The interval at which top queries should be emitted. + default: 60s + show_user: false + - name: top_query_plan_cache_size + type: integer + required: false + title: Query Plan Cache Size + description: The query plan cache size for storing explain query results. + default: 1000 + show_user: false + - name: top_query_plan_cache_ttl + type: duration + required: false + title: Query Plan Cache TTL + description: How long until a query plan expires in the cache. + default: 1h + show_user: false + - name: metrics + type: yaml + title: Metrics Configuration + description: Configuration for enabling optional metrics. + default: | + mysql.commands: + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + enabled: true + mysql.query.client.count: + enabled: true + mysql.table_open_cache: + enabled: true + mysql.client.network.io: + enabled: true + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.max_used_connections: + enabled: true + - name: data_stream.dataset + type: text + title: Dataset name + description: | + Dataset to write data to. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). + default: mysql_input_otel.generic + required: true + show_user: true +owner: + github: elastic/ecosystem + type: elastic diff --git a/test/packages/parallel/mysql_input_otel/sample_event_logs.json b/test/packages/parallel/mysql_input_otel/sample_event_logs.json new file mode 100644 index 0000000000..f68d431e56 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/sample_event_logs.json @@ -0,0 +1,121 @@ +{ + "@timestamp": "2026-04-07T13:27:54.607Z", + "attributes": { + "client": { + "address": "172.29.0.2", + "port": 48332 + }, + "db": { + "namespace": "", + "query": { + "text": "SELECT TABLE_SCHEMA, TABLE_NAME, COALESCE ( TABLE_ROWS, ? ), COALESCE ( AVG_ROW_LENGTH, ? ), COALESCE ( DATA_LENGTH, ? ), COALESCE ( INDEX_LENGTH, ? ) FROM information_schema.TABLES WHERE TABLE_SCHEMA NOT in ( ? )" + }, + "system": { + "name": "mysql" + } + }, + "mysql": { + "event_id": 5, + "events_statements_current": { + "digest": "d08f3a4d7e1be08b1fe479a1f9c7f56fc480b7b60c365cb330c65acd35ab1c42" + }, + "events_waits_current": { + "timer_wait": 0 + }, + "threads": { + "processlist_command": "Query", + "processlist_state": "Opening tables", + "thread_id": 52 + }, + "wait_type": "other" + }, + "network": { + "peer": { + "address": "172.29.0.2", + "port": 48332 + } + }, + "user": { + "name": "root" + } + }, + "client": { + "address": "172.29.0.2", + "port": 48332 + }, + "data_stream": { + "dataset": "mysqlreceiver.otel", + "namespace": "97979", + "type": "logs" + }, + "db": { + "namespace": "", + "query": { + "text": "SELECT TABLE_SCHEMA, TABLE_NAME, COALESCE ( TABLE_ROWS, ? ), COALESCE ( AVG_ROW_LENGTH, ? ), COALESCE ( DATA_LENGTH, ? ), COALESCE ( INDEX_LENGTH, ? ) FROM information_schema.TABLES WHERE TABLE_SCHEMA NOT in ( ? )" + }, + "system": { + "name": "mysql" + } + }, + "event": { + "dataset": "mysqlreceiver.otel" + }, + "event_name": "db.server.query_sample", + "host": { + "name": "elastic-agent-75030", + "os": { + "platform": "linux" + } + }, + "mysql": { + "event_id": 5, + "events_statements_current": { + "digest": "d08f3a4d7e1be08b1fe479a1f9c7f56fc480b7b60c365cb330c65acd35ab1c42" + }, + "events_waits_current": { + "timer_wait": 0 + }, + "threads": { + "processlist_command": "Query", + "processlist_state": "Opening tables", + "thread_id": 52 + }, + "wait_type": "other" + }, + "network": { + "peer": { + "address": "172.29.0.2", + "port": 48332 + } + }, + "observed_timestamp": "1970-01-01T00:00:00.000Z", + "os": { + "type": "linux" + }, + "resource": { + "attributes": { + "host": { + "name": "elastic-agent-75030" + }, + "os": { + "type": "linux" + } + }, + "schema_url": "https://opentelemetry.io/schemas/1.39.0" + }, + "scope": { + "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver", + "version": "9.4.0" + }, + "span": { + "id": "b5890df7e2f59ece" + }, + "span_id": "b5890df7e2f59ece", + "trace": { + "id": "c6236594c33ddc9a25f604748ced5b06" + }, + "trace_id": "c6236594c33ddc9a25f604748ced5b06", + "user": { + "name": "root" + } +} diff --git a/test/packages/parallel/mysql_input_otel/sample_event_metrics.json b/test/packages/parallel/mysql_input_otel/sample_event_metrics.json new file mode 100644 index 0000000000..e3108b0cc2 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel/sample_event_metrics.json @@ -0,0 +1,58 @@ +{ + "@timestamp": "2026-04-07T13:27:54.607Z", + "_metric_names_hash": "8402b118dcca946a", + "attributes": { + "kind": "merge_passes" + }, + "data_stream": { + "dataset": "mysqlreceiver.otel", + "namespace": "97979", + "type": "metrics" + }, + "event": { + "dataset": "mysqlreceiver.otel" + }, + "host": { + "name": "elastic-agent-75030", + "os": { + "platform": "linux" + } + }, + "kind": "merge_passes", + "metrics": { + "mysql": { + "sorts": 0 + } + }, + "mysql": { + "instance": { + "endpoint": "svc-mysql_input_otel:3306" + }, + "sorts": 0 + }, + "os": { + "type": "linux" + }, + "resource": { + "attributes": { + "host": { + "name": "elastic-agent-75030" + }, + "mysql": { + "instance": { + "endpoint": "svc-mysql_input_otel:3306" + } + }, + "os": { + "type": "linux" + } + }, + "schema_url": "https://opentelemetry.io/schemas/1.39.0" + }, + "scope": { + "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver", + "version": "9.4.0" + }, + "start_timestamp": "2026-04-07T13:27:53.591Z", + "unit": "1" +} From 6e8c589f250c0b802139514e32229f88f8b82c41 Mon Sep 17 00:00:00 2001 From: Tere Date: Wed, 8 Apr 2026 09:19:30 +0200 Subject: [PATCH 05/12] add stack version for mysql_input_otel package --- test/packages/parallel/mysql_input_otel.stack_version | 1 + 1 file changed, 1 insertion(+) create mode 100644 test/packages/parallel/mysql_input_otel.stack_version diff --git a/test/packages/parallel/mysql_input_otel.stack_version b/test/packages/parallel/mysql_input_otel.stack_version new file mode 100644 index 0000000000..b7ca1e13b0 --- /dev/null +++ b/test/packages/parallel/mysql_input_otel.stack_version @@ -0,0 +1 @@ +9.4.0-SNAPSHOT From 8013f746b1a534a15f6ae12aa3411eefb2e8f70b Mon Sep 17 00:00:00 2001 From: Tere Date: Wed, 8 Apr 2026 16:18:47 +0200 Subject: [PATCH 06/12] fix: avoid double .otel suffix and revert policyTemplate.Name derivation for dynamic packages BuildDataStreamName and expectedDatasets now apply the .otel suffix conditionally, preventing generic.otel from becoming generic.otel.otel when Fleet stores the real dataset value. The DynamicSignalTypes special case that derived the expected dataset from policyTemplate.Name is removed; scenario.dataStreamDataset is the source of truth for all otelcol input packages. Update policy test fixtures and sample events for mysql_input_otel, sql_server_input_otel, and zipkin_input_otel to reflect Fleet restoring data_stream.dataset in OTel transforms. Co-Authored-By: Claude Sonnet 4.6 --- internal/testrunner/runners/system/tester.go | 17 +-- .../testrunner/runners/system/tester_test.go | 122 ++++++++++++++++++ .../_dev/test/policy/test-default.expected | 2 + .../_dev/test/policy/test-default.yml | 3 +- .../mysql_input_otel/sample_event_logs.json | 30 +++-- .../sample_event_metrics.json | 26 ++-- .../test/policy/test-default-config.expected | 2 + .../sql_server_input_otel/sample_event.json | 22 ++-- .../sample_event_logs.json | 78 ++++++----- .../sample_event_metrics.json | 22 ++-- .../test-default-custom-dataset.expected | 1 + .../test-default-disable-use-apm.expected | 1 + .../test/policy/test-default-use-apm.expected | 1 + .../test/policy/test-default-vars.expected | 1 + .../_dev/test/policy/test-default.expected | 1 + .../zipkin_input_otel/sample_event_logs.json | 16 +-- .../sample_event_traces.json | 67 +++++----- 17 files changed, 282 insertions(+), 130 deletions(-) diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 6b4642af88..81b43b8300 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -1486,7 +1486,9 @@ func BuildDataStreamName(dsType, dsDataset, namespace string, policyTemplate pac // Input packages using the otel collector input require to add a specific dataset suffix if packageType == "input" && policyTemplate.Input == otelCollectorInputName { - dataset = fmt.Sprintf("%s.%s", dataset, otelSuffixDataset) + if !strings.HasSuffix(dataset, "."+otelSuffixDataset) { + dataset = dataset + "." + otelSuffixDataset + } } return fmt.Sprintf("%s-%s-%s", dsType, dataset, namespace) @@ -1974,15 +1976,10 @@ func (r *tester) expectedDatasets(scenario *scenarioTest, config *testConfig) ([ // get dataset directly from package policy added when preparing the scenario expectedDataset := scenario.dataStreamDataset if scenario.policyTemplate.Input == otelCollectorInputName { - if scenario.policyTemplate.DynamicSignalTypes { - // For dynamic_signal_types packages, Fleet stores "generic.otel" as the - // policy dataset, but the OTel collector routes documents to scope-based - // data streams (e.g. .otel). Derive the expected - // dataset from the policy template name, not the policy variable. - expectedDataset = scenario.policyTemplate.Name + "." + otelSuffixDataset - } else { - // Input packages whose input is `otelcol` must add the `.otel` suffix - // Example: httpcheck.metrics.otel + // Input packages whose input is `otelcol` must add the `.otel` suffix. + // The suffix is conditional to avoid double-appending when the stored dataset + // already ends in `.otel` (e.g. "generic.otel"). + if !strings.HasSuffix(expectedDataset, "."+otelSuffixDataset) { expectedDataset += "." + otelSuffixDataset } // Traces can also emit to a shared logs data stream (e.g. logs-generic.otel-*). diff --git a/internal/testrunner/runners/system/tester_test.go b/internal/testrunner/runners/system/tester_test.go index cfd1720721..90bf77058e 100644 --- a/internal/testrunner/runners/system/tester_test.go +++ b/internal/testrunner/runners/system/tester_test.go @@ -503,6 +503,128 @@ func TestDiscoverDataStreams(t *testing.T) { }) } +func TestBuildDataStreamName(t *testing.T) { + cases := []struct { + title string + dsType string + dsDataset string + namespace string + policyTemplate packages.PolicyTemplate + packageType string + expected string + }{ + { + title: "non-otelcol input: no suffix added", + dsType: "logs", + dsDataset: "nginx.access", + namespace: "default", + policyTemplate: packages.PolicyTemplate{Input: "logfile"}, + packageType: "integration", + expected: "logs-nginx.access-default", + }, + { + title: "otelcol input: .otel suffix appended", + dsType: "logs", + dsDataset: "httpcheck", + namespace: "default", + policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, + packageType: "input", + expected: "logs-httpcheck.otel-default", + }, + { + title: "otelcol input: no double-suffix when dataset already ends in .otel", + dsType: "logs", + dsDataset: "generic.otel", + namespace: "default", + policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, + packageType: "input", + expected: "logs-generic.otel-default", + }, + { + title: "otelcol input on integration package type: no suffix added", + dsType: "metrics", + dsDataset: "myreceiver", + namespace: "default", + policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, + packageType: "integration", + expected: "metrics-myreceiver-default", + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + got := BuildDataStreamName(c.dsType, c.dsDataset, c.namespace, c.policyTemplate, c.packageType) + assert.Equal(t, c.expected, got) + }) + } +} + +func TestExpectedDatasets(t *testing.T) { + cases := []struct { + title string + scenario *scenarioTest + expected []string + }{ + { + title: "non-otelcol package: dataset returned as-is", + scenario: &scenarioTest{ + dataStreamDataset: "nginx.access", + policyTemplate: packages.PolicyTemplate{Input: "logfile"}, + }, + expected: []string{"nginx.access"}, + }, + { + title: "otelcol input: .otel suffix appended and generic.otel included", + scenario: &scenarioTest{ + dataStreamDataset: "httpcheck", + policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, + }, + expected: []string{"httpcheck.otel", "generic.otel"}, + }, + { + title: "otelcol input with generic.otel dataset: no double-suffix", + scenario: &scenarioTest{ + dataStreamDataset: "generic.otel", + policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, + }, + expected: []string{"generic.otel", "generic.otel"}, + }, + { + title: "otelcol dynamic_signal_types: uses stored dataset, not policyTemplate.Name", + scenario: &scenarioTest{ + dataStreamDataset: "sqlserverreceiver", + policyTemplate: packages.PolicyTemplate{ + Name: "sqlserverreceiver", + Input: otelCollectorInputName, + DynamicSignalTypes: true, + }, + }, + expected: []string{"sqlserverreceiver.otel", "generic.otel"}, + }, + { + title: "otelcol dynamic_signal_types with generic.otel dataset: no double-suffix", + scenario: &scenarioTest{ + dataStreamDataset: "generic.otel", + policyTemplate: packages.PolicyTemplate{ + Name: "sqlserverreceiver", + Input: otelCollectorInputName, + DynamicSignalTypes: true, + }, + }, + expected: []string{"generic.otel", "generic.otel"}, + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + r := &tester{} + got, err := r.expectedDatasets(c.scenario, nil) + require.NoError(t, err) + assert.Equal(t, c.expected, got) + }) + } +} + func TestBuildDataStreamScenarios(t *testing.T) { t.Run("standard single stream", func(t *testing.T) { r := &tester{pkgManifest: &packages.PackageManifest{Type: "integration"}} diff --git a/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected b/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected index 3a727a0b89..3840154aaf 100644 --- a/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected +++ b/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected @@ -33,11 +33,13 @@ processors: - context: log statements: - set(attributes["data_stream.type"], "logs") + - set(attributes["data_stream.dataset"], "mysql_input_otel.generic") - set(attributes["data_stream.namespace"], "ep") metric_statements: - context: datapoint statements: - set(attributes["data_stream.type"], "metrics") + - set(attributes["data_stream.dataset"], "mysql_input_otel.generic") - set(attributes["data_stream.namespace"], "ep") receivers: mysql/componentid-0: diff --git a/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.yml b/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.yml index 12afef89c9..f057622ddd 100644 --- a/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.yml +++ b/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.yml @@ -1,2 +1 @@ -vars: - data_stream.dataset: mysql_input_otel.test +vars: ~ \ No newline at end of file diff --git a/test/packages/parallel/mysql_input_otel/sample_event_logs.json b/test/packages/parallel/mysql_input_otel/sample_event_logs.json index f68d431e56..a1a7ea1c2e 100644 --- a/test/packages/parallel/mysql_input_otel/sample_event_logs.json +++ b/test/packages/parallel/mysql_input_otel/sample_event_logs.json @@ -1,9 +1,9 @@ { - "@timestamp": "2026-04-07T13:27:54.607Z", + "@timestamp": "2026-04-08T13:54:17.208Z", "attributes": { "client": { "address": "172.29.0.2", - "port": 48332 + "port": 39480 }, "db": { "namespace": "", @@ -32,7 +32,7 @@ "network": { "peer": { "address": "172.29.0.2", - "port": 48332 + "port": 39480 } }, "user": { @@ -41,11 +41,11 @@ }, "client": { "address": "172.29.0.2", - "port": 48332 + "port": 39480 }, "data_stream": { - "dataset": "mysqlreceiver.otel", - "namespace": "97979", + "dataset": "mysql_input_otel.generic.otel", + "namespace": "53297", "type": "logs" }, "db": { @@ -58,11 +58,13 @@ } }, "event": { - "dataset": "mysqlreceiver.otel" + "agent_id_status": "missing", + "dataset": "mysql_input_otel.generic.otel", + "ingested": "2026-04-08T13:54:27Z" }, "event_name": "db.server.query_sample", "host": { - "name": "elastic-agent-75030", + "name": "elastic-agent-23953", "os": { "platform": "linux" } @@ -85,7 +87,7 @@ "network": { "peer": { "address": "172.29.0.2", - "port": 48332 + "port": 39480 } }, "observed_timestamp": "1970-01-01T00:00:00.000Z", @@ -95,7 +97,7 @@ "resource": { "attributes": { "host": { - "name": "elastic-agent-75030" + "name": "elastic-agent-23953" }, "os": { "type": "linux" @@ -108,13 +110,13 @@ "version": "9.4.0" }, "span": { - "id": "b5890df7e2f59ece" + "id": "8439d48bb0ce6738" }, - "span_id": "b5890df7e2f59ece", + "span_id": "8439d48bb0ce6738", "trace": { - "id": "c6236594c33ddc9a25f604748ced5b06" + "id": "2c3069c1a55670bee4d912639a0e184c" }, - "trace_id": "c6236594c33ddc9a25f604748ced5b06", + "trace_id": "2c3069c1a55670bee4d912639a0e184c", "user": { "name": "root" } diff --git a/test/packages/parallel/mysql_input_otel/sample_event_metrics.json b/test/packages/parallel/mysql_input_otel/sample_event_metrics.json index e3108b0cc2..ed22eb4495 100644 --- a/test/packages/parallel/mysql_input_otel/sample_event_metrics.json +++ b/test/packages/parallel/mysql_input_otel/sample_event_metrics.json @@ -1,34 +1,35 @@ { - "@timestamp": "2026-04-07T13:27:54.607Z", - "_metric_names_hash": "8402b118dcca946a", + "@timestamp": "2026-04-08T13:54:17.208Z", + "_metric_names_hash": "800d765540031faf", "attributes": { - "kind": "merge_passes" + "status": "overflow" }, "data_stream": { - "dataset": "mysqlreceiver.otel", - "namespace": "97979", + "dataset": "mysql_input_otel.generic.otel", + "namespace": "53297", "type": "metrics" }, "event": { - "dataset": "mysqlreceiver.otel" + "agent_id_status": "missing", + "dataset": "mysql_input_otel.generic.otel", + "ingested": "2026-04-08T13:54:27Z" }, "host": { - "name": "elastic-agent-75030", + "name": "elastic-agent-23953", "os": { "platform": "linux" } }, - "kind": "merge_passes", "metrics": { "mysql": { - "sorts": 0 + "table_open_cache": 0 } }, "mysql": { "instance": { "endpoint": "svc-mysql_input_otel:3306" }, - "sorts": 0 + "table_open_cache": 0 }, "os": { "type": "linux" @@ -36,7 +37,7 @@ "resource": { "attributes": { "host": { - "name": "elastic-agent-75030" + "name": "elastic-agent-23953" }, "mysql": { "instance": { @@ -53,6 +54,7 @@ "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver", "version": "9.4.0" }, - "start_timestamp": "2026-04-07T13:27:53.591Z", + "start_timestamp": "2026-04-08T13:54:16.199Z", + "status": "overflow", "unit": "1" } diff --git a/test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.expected b/test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.expected index fc9d284b0f..5d2af76211 100644 --- a/test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.expected +++ b/test/packages/parallel/sql_server_input_otel/_dev/test/policy/test-default-config.expected @@ -33,11 +33,13 @@ processors: - context: log statements: - set(attributes["data_stream.type"], "logs") + - set(attributes["data_stream.dataset"], "sqlserverreceiver") - set(attributes["data_stream.namespace"], "ep") metric_statements: - context: datapoint statements: - set(attributes["data_stream.type"], "metrics") + - set(attributes["data_stream.dataset"], "sqlserverreceiver") - set(attributes["data_stream.namespace"], "ep") receivers: sqlserver/componentid-0: diff --git a/test/packages/parallel/sql_server_input_otel/sample_event.json b/test/packages/parallel/sql_server_input_otel/sample_event.json index 2e15c06e72..b03f7efec9 100644 --- a/test/packages/parallel/sql_server_input_otel/sample_event.json +++ b/test/packages/parallel/sql_server_input_otel/sample_event.json @@ -1,21 +1,23 @@ { - "@timestamp": "2026-04-07T11:03:39.789Z", + "@timestamp": "2026-04-08T13:51:30.682Z", "_metric_names_hash": "7ee276998c966647", "attributes": { "performance_counter": { - "object_name": "SQLServer:Buffer Node" + "object_name": "SQLServer:Buffer Manager" } }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "23377", + "namespace": "24063", "type": "metrics" }, "event": { - "dataset": "sqlserverreceiver.otel" + "agent_id_status": "missing", + "dataset": "sqlserverreceiver.otel", + "ingested": "2026-04-08T13:51:40Z" }, "host": { - "name": "elastic-agent-57639", + "name": "elastic-agent-75226", "os": { "platform": "linux" } @@ -23,7 +25,7 @@ "metrics": { "sqlserver": { "page": { - "life_expectancy": 41 + "life_expectancy": 16 } } }, @@ -31,12 +33,12 @@ "type": "linux" }, "performance_counter": { - "object_name": "SQLServer:Buffer Node" + "object_name": "SQLServer:Buffer Manager" }, "resource": { "attributes": { "host": { - "name": "elastic-agent-57639" + "name": "elastic-agent-75226" }, "os": { "type": "linux" @@ -63,9 +65,9 @@ }, "sqlserver": { "page": { - "life_expectancy": 41 + "life_expectancy": 16 } }, - "start_timestamp": "2026-04-07T11:03:38.605Z", + "start_timestamp": "2026-04-08T13:51:29.559Z", "unit": "s" } diff --git a/test/packages/parallel/sql_server_input_otel/sample_event_logs.json b/test/packages/parallel/sql_server_input_otel/sample_event_logs.json index 6699b0b2c8..db1ebd67ba 100644 --- a/test/packages/parallel/sql_server_input_otel/sample_event_logs.json +++ b/test/packages/parallel/sql_server_input_otel/sample_event_logs.json @@ -1,12 +1,15 @@ { - "@timestamp": "2026-04-07T11:01:48.355Z", + "@timestamp": "2026-04-08T13:50:11.062Z", "attributes": { "client": { - "address": "elastic-agent-10224", - "port": 48948 + "address": "elastic-agent-71667", + "port": 32812 }, "db": { "namespace": "master", + "query": { + "text": "WITH PerfCounters SELECT DISTINCT RTRIM ( spi. object_name ) object_name, RTRIM ( spi. counter_name ) counter_name, RTRIM ( spi. instance_name ), CAST ( spi. cntr_value ), spi. cntr_type FROM sys.dm_os_performance_counters WHERE counter_name IN ( ? ) OR ( spi. object_name LIKE ? OR spi. object_name LIKE ? OR spi. object_name LIKE ? ) OR ( spi. instance_name IN ( ? ) AND spi. counter_name IN ( ? ) ) ) INSERT INTO @PCounters SELECT * FROM PerfCounters" + }, "system": { "name": "microsoft.sql_server" } @@ -14,63 +17,68 @@ "network": { "peer": { "address": "172.29.0.2", - "port": 48948 + "port": 32812 } }, "sqlserver": { "blocking_session_id": 0, - "command": "SELECT", + "command": "INSERT", "context_info": "", - "cpu_time": 0.046, - "deadlock_priority": 0, + "cpu_time": 0.052, + "deadlock_priority": -10, "estimated_completion_time": 0, "lock_timeout": -0.001, - "logical_reads": 669, + "logical_reads": 607, "open_transaction_count": 0, "percent_complete": 0, "procedure_id": "0", "procedure_name": "", - "query_hash": "30447d7d5ef3066a", - "query_plan_hash": "0c97dc34ff850504", - "query_start": "2026-04-07T11:01:48.297+00:00", - "reads": 104, + "query_hash": "6df033ef55623a4a", + "query_plan_hash": "49c16c014d7e4505", + "query_start": "2026-04-08T13:50:10.997+00:00", + "reads": 40, "request_status": "running", - "row_count": 0, + "row_count": 1, "session_id": 51, "session_status": "running", - "total_elapsed_time": 0.047, - "transaction_id": 1899, + "total_elapsed_time": 0.053, + "transaction_id": 0, "transaction_isolation_level": 2, "wait_resource": "", "wait_time": 0, "wait_type": "", - "writes": 0 + "writes": 19 }, "user": { "name": "sa" } }, "client": { - "address": "elastic-agent-10224", - "port": 48948 + "address": "elastic-agent-71667", + "port": 32812 }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "19985", + "namespace": "78202", "type": "logs" }, "db": { "namespace": "master", + "query": { + "text": "WITH PerfCounters SELECT DISTINCT RTRIM ( spi. object_name ) object_name, RTRIM ( spi. counter_name ) counter_name, RTRIM ( spi. instance_name ), CAST ( spi. cntr_value ), spi. cntr_type FROM sys.dm_os_performance_counters WHERE counter_name IN ( ? ) OR ( spi. object_name LIKE ? OR spi. object_name LIKE ? OR spi. object_name LIKE ? ) OR ( spi. instance_name IN ( ? ) AND spi. counter_name IN ( ? ) ) ) INSERT INTO @PCounters SELECT * FROM PerfCounters" + }, "system": { "name": "microsoft.sql_server" } }, "event": { - "dataset": "sqlserverreceiver.otel" + "agent_id_status": "missing", + "dataset": "sqlserverreceiver.otel", + "ingested": "2026-04-08T13:50:21Z" }, "event_name": "db.server.query_sample", "host": { - "name": "elastic-agent-10224", + "name": "elastic-agent-71667", "os": { "platform": "linux" } @@ -78,7 +86,7 @@ "network": { "peer": { "address": "172.29.0.2", - "port": 48948 + "port": 32812 } }, "observed_timestamp": "1970-01-01T00:00:00.000Z", @@ -88,7 +96,7 @@ "resource": { "attributes": { "host": { - "name": "elastic-agent-10224" + "name": "elastic-agent-71667" }, "os": { "type": "linux" @@ -115,32 +123,32 @@ }, "sqlserver": { "blocking_session_id": 0, - "command": "SELECT", + "command": "INSERT", "context_info": "", - "cpu_time": 0.046, - "deadlock_priority": 0, + "cpu_time": 0.052, + "deadlock_priority": -10, "estimated_completion_time": 0, "lock_timeout": -0.001, - "logical_reads": 669, + "logical_reads": 607, "open_transaction_count": 0, "percent_complete": 0, "procedure_id": "0", "procedure_name": "", - "query_hash": "30447d7d5ef3066a", - "query_plan_hash": "0c97dc34ff850504", - "query_start": "2026-04-07T11:01:48.297+00:00", - "reads": 104, + "query_hash": "6df033ef55623a4a", + "query_plan_hash": "49c16c014d7e4505", + "query_start": "2026-04-08T13:50:10.997+00:00", + "reads": 40, "request_status": "running", - "row_count": 0, + "row_count": 1, "session_id": 51, "session_status": "running", - "total_elapsed_time": 0.047, - "transaction_id": 1899, + "total_elapsed_time": 0.053, + "transaction_id": 0, "transaction_isolation_level": 2, "wait_resource": "", "wait_time": 0, "wait_type": "", - "writes": 0 + "writes": 19 }, "user": { "name": "sa" diff --git a/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json b/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json index f132f8a3a0..dbf4134d14 100644 --- a/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json +++ b/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json @@ -1,21 +1,23 @@ { - "@timestamp": "2026-04-07T11:01:48.409Z", + "@timestamp": "2026-04-08T13:50:11.123Z", "_metric_names_hash": "7ee276998c966647", "attributes": { "performance_counter": { - "object_name": "SQLServer:Buffer Node" + "object_name": "SQLServer:Buffer Manager" } }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "19985", + "namespace": "78202", "type": "metrics" }, "event": { - "dataset": "sqlserverreceiver.otel" + "agent_id_status": "missing", + "dataset": "sqlserverreceiver.otel", + "ingested": "2026-04-08T13:50:21Z" }, "host": { - "name": "elastic-agent-10224", + "name": "elastic-agent-71667", "os": { "platform": "linux" } @@ -23,7 +25,7 @@ "metrics": { "sqlserver": { "page": { - "life_expectancy": 22 + "life_expectancy": 17 } } }, @@ -31,12 +33,12 @@ "type": "linux" }, "performance_counter": { - "object_name": "SQLServer:Buffer Node" + "object_name": "SQLServer:Buffer Manager" }, "resource": { "attributes": { "host": { - "name": "elastic-agent-10224" + "name": "elastic-agent-71667" }, "os": { "type": "linux" @@ -63,9 +65,9 @@ }, "sqlserver": { "page": { - "life_expectancy": 22 + "life_expectancy": 17 } }, - "start_timestamp": "2026-04-07T11:01:47.281Z", + "start_timestamp": "2026-04-08T13:50:09.970Z", "unit": "s" } diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected index 735a92b5d0..cddb652cb4 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected @@ -42,6 +42,7 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") + - set(attributes["data_stream.dataset"], "zipkin.custom") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected index 3f6b7eb6fc..9bfab820c3 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected @@ -30,6 +30,7 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") + - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected index 735a92b5d0..005a7c72b6 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected @@ -42,6 +42,7 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") + - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected index ab89a91b4c..c23cff1583 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected @@ -42,6 +42,7 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") + - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected index 735a92b5d0..005a7c72b6 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected @@ -42,6 +42,7 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") + - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/sample_event_logs.json b/test/packages/parallel/zipkin_input_otel/sample_event_logs.json index 8549006d1f..c7a7f24fb4 100644 --- a/test/packages/parallel/zipkin_input_otel/sample_event_logs.json +++ b/test/packages/parallel/zipkin_input_otel/sample_event_logs.json @@ -1,5 +1,5 @@ { - "@timestamp": "2026-04-07T10:57:35.581Z", + "@timestamp": "2026-04-08T13:39:36.343Z", "agent": { "name": "otlp", "version": "unknown" @@ -9,12 +9,12 @@ "name": "wr" }, "timestamp": { - "us": 1775559455581041 + "us": 1775655576343981 } }, "data_stream": { "dataset": "generic.otel", - "namespace": "88673", + "namespace": "73913", "type": "logs" }, "event": { @@ -37,14 +37,14 @@ "name": "backend" }, "span": { - "id": "e86a426e61162cce" + "id": "ffa0fe740327e8db" }, - "span_id": "e86a426e61162cce", + "span_id": "ffa0fe740327e8db", "timestamp": { - "us": 1775559455581041 + "us": 1775655576343981 }, "trace": { - "id": "0000000000000000e86a426e61162cce" + "id": "0000000000000000ffa0fe740327e8db" }, - "trace_id": "0000000000000000e86a426e61162cce" + "trace_id": "0000000000000000ffa0fe740327e8db" } diff --git a/test/packages/parallel/zipkin_input_otel/sample_event_traces.json b/test/packages/parallel/zipkin_input_otel/sample_event_traces.json index 2011a36fff..d9be3f953a 100644 --- a/test/packages/parallel/zipkin_input_otel/sample_event_traces.json +++ b/test/packages/parallel/zipkin_input_otel/sample_event_traces.json @@ -1,17 +1,19 @@ { - "@timestamp": "2026-04-07T10:57:35.580Z", + "@timestamp": "2026-04-08T13:39:36.343Z", "agent": { "name": "otlp", "version": "unknown" }, "attributes": { + "error": "404", "event": { - "outcome": "success", - "success_count": 1 + "outcome": "failure", + "success_count": 0 }, "http": { "method": "GET", - "path": "/api" + "path": "/", + "status_code": "404" }, "net": { "host": { @@ -19,52 +21,56 @@ }, "peer": { "ip": "172.30.0.3", - "port": 45894 + "port": 49798 } }, "processor": { "event": "transaction" }, "timestamp": { - "us": 1775559455580995 + "us": 1775655576343934 }, "transaction": { "duration": { - "us": 709 + "us": 1128 }, - "id": "e86a426e61162cce", - "name": "GET /api", + "id": "ffa0fe740327e8db", + "name": "GET /*", "representative_count": 1, - "result": "Success", + "result": "Error", "root": true, "sampled": true, "type": "request" } }, "data_stream": { - "dataset": "generic.otel", - "namespace": "88673", + "dataset": "zipkinreceiver.otel", + "namespace": "73913", "type": "traces" }, - "duration": 709000, + "duration": 1128000, + "error": "404", "event": { - "dataset": "generic.otel", - "outcome": "success", - "success_count": 1 + "agent_id_status": "missing", + "dataset": "zipkinreceiver.otel", + "ingested": "2026-04-08T13:39:46Z", + "outcome": "failure", + "success_count": 0 }, "http": { "method": "GET", - "path": "/api" + "path": "/", + "status_code": "404" }, "kind": "Server", - "name": "GET /api", + "name": "GET /*", "net": { "host": { "ip": "172.30.0.2" }, "peer": { "ip": "172.30.0.3", - "port": 45894 + "port": 49798 } }, "processor": { @@ -85,25 +91,28 @@ "name": "backend" }, "span": { - "id": "e86a426e61162cce", - "name": "GET /api" + "id": "ffa0fe740327e8db", + "name": "GET /*" + }, + "span_id": "ffa0fe740327e8db", + "status": { + "code": "Error" }, - "span_id": "e86a426e61162cce", "timestamp": { - "us": 1775559455580995 + "us": 1775655576343934 }, "trace": { - "id": "0000000000000000e86a426e61162cce" + "id": "0000000000000000ffa0fe740327e8db" }, - "trace_id": "0000000000000000e86a426e61162cce", + "trace_id": "0000000000000000ffa0fe740327e8db", "transaction": { "duration": { - "us": 709 + "us": 1128 }, - "id": "e86a426e61162cce", - "name": "GET /api", + "id": "ffa0fe740327e8db", + "name": "GET /*", "representative_count": 1, - "result": "Success", + "result": "Error", "root": true, "sampled": true, "type": "request" From 6c0f63a29da7946b9ff94445bed1bfaba554f4f2 Mon Sep 17 00:00:00 2001 From: Tere Date: Wed, 8 Apr 2026 16:47:24 +0200 Subject: [PATCH 07/12] fix: stop implicitly validating generic.otel for otelcol input packages Made-with: Cursor --- internal/kibana/packagepolicy.go | 8 -------- internal/kibana/packagepolicy_test.go | 10 +++------- .../testdata/otel_input_dynamic_signals.json | 4 ++-- .../otel_input_dynamic_signals_legacy.json | 6 +++--- .../packages/otel_dynamic_input/manifest.yml | 4 +++- internal/testrunner/runners/system/tester.go | 5 ++--- .../testrunner/runners/system/tester_test.go | 18 +++++++++--------- 7 files changed, 22 insertions(+), 33 deletions(-) diff --git a/internal/kibana/packagepolicy.go b/internal/kibana/packagepolicy.go index 420b0b3f8a..3e2969cbee 100644 --- a/internal/kibana/packagepolicy.go +++ b/internal/kibana/packagepolicy.go @@ -161,12 +161,7 @@ func BuildInputPackagePolicy( } // Dataset key for the stream: .. - // For packages with dynamic_signal_types, Fleet uses "generic.otel" as the - // stream path, so we must match that key in the simplified API request. streamDataset := fmt.Sprintf("%s.%s", manifest.Name, policyTemplate.Name) - if policyTemplate.DynamicSignalTypes { - streamDataset = "generic.otel" - } vars := SetKibanaVariables(policyTemplate.Vars, varValues) ensureDatasetVar(vars, policyTemplate, varValues) @@ -240,9 +235,6 @@ func ensureDatasetVar(vars Vars, policyTemplate packages.PolicyTemplate, varValu return } dataset := policyTemplate.Name - if policyTemplate.DynamicSignalTypes { - dataset = "generic.otel" - } for _, def := range policyTemplate.Vars { if def.Name == "data_stream.dataset" && def.Default != nil { if s, ok := def.Default.Value().(string); ok && s != "" { diff --git a/internal/kibana/packagepolicy_test.go b/internal/kibana/packagepolicy_test.go index fd2f6fe14f..29c402c2fb 100644 --- a/internal/kibana/packagepolicy_test.go +++ b/internal/kibana/packagepolicy_test.go @@ -283,11 +283,7 @@ func TestBuildInputPackagePolicy(t *testing.T) { goldenLegacy: "testdata/otel_traces_use_apm_legacy.json", }, { - // Input package with dynamic_signal_types: true. Fleet now uses - // "generic.otel" as the stream path/dataset for such packages, so - // BuildInputPackagePolicy must use "generic.otel" as the stream key - // and the data_stream.dataset variable value. - name: "otel_input_dynamic_signals", + name: "otel_dynamic_signal_types_default_dataset", packageRoot: "testdata/packages/otel_dynamic_input", policyTemplateName: "sqlreceiver", policyName: "otel-dynamic-test", @@ -529,11 +525,11 @@ func TestEnsureDatasetVar(t *testing.T) { wantDataset: "sql_query", }, { - name: "dynamic_signal_types uses generic.otel", + name: "dynamic_signal_types falls back to policy template name", vars: Vars{}, policyTemplate: packages.PolicyTemplate{Name: "sqlreceiver", DynamicSignalTypes: true}, varValues: common.MapStr{}, - wantDataset: "generic.otel", + wantDataset: "sqlreceiver", }, } diff --git a/internal/kibana/testdata/otel_input_dynamic_signals.json b/internal/kibana/testdata/otel_input_dynamic_signals.json index be15b54229..b382a07b69 100644 --- a/internal/kibana/testdata/otel_input_dynamic_signals.json +++ b/internal/kibana/testdata/otel_input_dynamic_signals.json @@ -11,10 +11,10 @@ "sqlreceiver-otelcol": { "enabled": true, "streams": { - "generic.otel": { + "otel_dynamic_input.sqlreceiver": { "enabled": true, "vars": { - "data_stream.dataset": "generic.otel" + "data_stream.dataset": "sqlreceiver" } } } diff --git a/internal/kibana/testdata/otel_input_dynamic_signals_legacy.json b/internal/kibana/testdata/otel_input_dynamic_signals_legacy.json index 70cf183578..c3a69a24ce 100644 --- a/internal/kibana/testdata/otel_input_dynamic_signals_legacy.json +++ b/internal/kibana/testdata/otel_input_dynamic_signals_legacy.json @@ -16,7 +16,7 @@ "enabled": true, "vars": { "data_stream.dataset": { - "value": "generic.otel", + "value": "sqlreceiver", "type": "text" } }, @@ -25,11 +25,11 @@ "enabled": true, "data_stream": { "type": "", - "dataset": "generic.otel" + "dataset": "otel_dynamic_input.sqlreceiver" }, "vars": { "data_stream.dataset": { - "value": "generic.otel", + "value": "sqlreceiver", "type": "text" } } diff --git a/internal/kibana/testdata/packages/otel_dynamic_input/manifest.yml b/internal/kibana/testdata/packages/otel_dynamic_input/manifest.yml index a347eaead1..33e06ce086 100644 --- a/internal/kibana/testdata/packages/otel_dynamic_input/manifest.yml +++ b/internal/kibana/testdata/packages/otel_dynamic_input/manifest.yml @@ -3,7 +3,9 @@ name: otel_dynamic_input title: OTel Dynamic Signals Input Package description: >- Minimal OTel input package with dynamic_signal_types used to verify that - BuildInputPackagePolicy uses "generic.otel" as the stream key. + BuildInputPackagePolicy uses the "{package}.{policy_template}" stream key and + resolves data_stream.dataset from user vars, manifest defaults, or the policy + template name. type: input version: 0.1.0 categories: diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 81b43b8300..19e7a8c44d 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -1978,12 +1978,11 @@ func (r *tester) expectedDatasets(scenario *scenarioTest, config *testConfig) ([ if scenario.policyTemplate.Input == otelCollectorInputName { // Input packages whose input is `otelcol` must add the `.otel` suffix. // The suffix is conditional to avoid double-appending when the stored dataset - // already ends in `.otel` (e.g. "generic.otel"). + // already ends in `.otel` (user-configured via data_stream.dataset). if !strings.HasSuffix(expectedDataset, "."+otelSuffixDataset) { expectedDataset += "." + otelSuffixDataset } - // Traces can also emit to a shared logs data stream (e.g. logs-generic.otel-*). - expectedDatasets = []string{expectedDataset, "generic." + otelSuffixDataset} + expectedDatasets = []string{expectedDataset} } else { expectedDatasets = []string{expectedDataset} } diff --git a/internal/testrunner/runners/system/tester_test.go b/internal/testrunner/runners/system/tester_test.go index 90bf77058e..dd1211d4f8 100644 --- a/internal/testrunner/runners/system/tester_test.go +++ b/internal/testrunner/runners/system/tester_test.go @@ -534,11 +534,11 @@ func TestBuildDataStreamName(t *testing.T) { { title: "otelcol input: no double-suffix when dataset already ends in .otel", dsType: "logs", - dsDataset: "generic.otel", + dsDataset: "custom.otel", namespace: "default", policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, packageType: "input", - expected: "logs-generic.otel-default", + expected: "logs-custom.otel-default", }, { title: "otelcol input on integration package type: no suffix added", @@ -574,20 +574,20 @@ func TestExpectedDatasets(t *testing.T) { expected: []string{"nginx.access"}, }, { - title: "otelcol input: .otel suffix appended and generic.otel included", + title: "otelcol input: .otel suffix appended", scenario: &scenarioTest{ dataStreamDataset: "httpcheck", policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, }, - expected: []string{"httpcheck.otel", "generic.otel"}, + expected: []string{"httpcheck.otel"}, }, { - title: "otelcol input with generic.otel dataset: no double-suffix", + title: "otelcol input with user dataset already ending in .otel: no double-suffix", scenario: &scenarioTest{ dataStreamDataset: "generic.otel", policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, }, - expected: []string{"generic.otel", "generic.otel"}, + expected: []string{"generic.otel"}, }, { title: "otelcol dynamic_signal_types: uses stored dataset, not policyTemplate.Name", @@ -599,10 +599,10 @@ func TestExpectedDatasets(t *testing.T) { DynamicSignalTypes: true, }, }, - expected: []string{"sqlserverreceiver.otel", "generic.otel"}, + expected: []string{"sqlserverreceiver.otel"}, }, { - title: "otelcol dynamic_signal_types with generic.otel dataset: no double-suffix", + title: "otelcol dynamic_signal_types with user dataset already ending in .otel: no double-suffix", scenario: &scenarioTest{ dataStreamDataset: "generic.otel", policyTemplate: packages.PolicyTemplate{ @@ -611,7 +611,7 @@ func TestExpectedDatasets(t *testing.T) { DynamicSignalTypes: true, }, }, - expected: []string{"generic.otel", "generic.otel"}, + expected: []string{"generic.otel"}, }, } From 4002c43b1f6487b2e8a531d37fa13e0c63147782 Mon Sep 17 00:00:00 2001 From: Tere Date: Wed, 8 Apr 2026 17:01:14 +0200 Subject: [PATCH 08/12] fix: update example event in README for sql_server_input_otel --- .../sql_server_input_otel/docs/README.md | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/test/packages/parallel/sql_server_input_otel/docs/README.md b/test/packages/parallel/sql_server_input_otel/docs/README.md index 61c6281272..53d787762b 100644 --- a/test/packages/parallel/sql_server_input_otel/docs/README.md +++ b/test/packages/parallel/sql_server_input_otel/docs/README.md @@ -57,23 +57,25 @@ An example event looks as following: ```json { - "@timestamp": "2026-04-07T11:03:39.789Z", + "@timestamp": "2026-04-08T13:51:30.682Z", "_metric_names_hash": "7ee276998c966647", "attributes": { "performance_counter": { - "object_name": "SQLServer:Buffer Node" + "object_name": "SQLServer:Buffer Manager" } }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "23377", + "namespace": "24063", "type": "metrics" }, "event": { - "dataset": "sqlserverreceiver.otel" + "agent_id_status": "missing", + "dataset": "sqlserverreceiver.otel", + "ingested": "2026-04-08T13:51:40Z" }, "host": { - "name": "elastic-agent-57639", + "name": "elastic-agent-75226", "os": { "platform": "linux" } @@ -81,7 +83,7 @@ An example event looks as following: "metrics": { "sqlserver": { "page": { - "life_expectancy": 41 + "life_expectancy": 16 } } }, @@ -89,12 +91,12 @@ An example event looks as following: "type": "linux" }, "performance_counter": { - "object_name": "SQLServer:Buffer Node" + "object_name": "SQLServer:Buffer Manager" }, "resource": { "attributes": { "host": { - "name": "elastic-agent-57639" + "name": "elastic-agent-75226" }, "os": { "type": "linux" @@ -121,10 +123,10 @@ An example event looks as following: }, "sqlserver": { "page": { - "life_expectancy": 41 + "life_expectancy": 16 } }, - "start_timestamp": "2026-04-07T11:03:38.605Z", + "start_timestamp": "2026-04-08T13:51:29.559Z", "unit": "s" } ``` From b7943cd02f9e4bd56b3a608587c8029c7511dc78 Mon Sep 17 00:00:00 2001 From: Tere Date: Fri, 10 Apr 2026 11:50:08 +0200 Subject: [PATCH 09/12] Refactor expectedDatasets function to remove unnecessary nil parameter and simplify logic. --- internal/testrunner/runners/system/tester.go | 4 +--- internal/testrunner/runners/system/tester_test.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 81a3423de7..4194d40d09 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -1982,10 +1982,8 @@ func (r *tester) expectedDatasets(scenario *scenarioTest) ([]string, error) { if !strings.HasSuffix(expectedDataset, "."+otelSuffixDataset) { expectedDataset += "." + otelSuffixDataset } - expectedDatasets = []string{expectedDataset} - } else { - expectedDatasets = []string{expectedDataset} } + expectedDatasets = []string{expectedDataset} } return expectedDatasets, nil diff --git a/internal/testrunner/runners/system/tester_test.go b/internal/testrunner/runners/system/tester_test.go index dd1211d4f8..4f7c6e97a5 100644 --- a/internal/testrunner/runners/system/tester_test.go +++ b/internal/testrunner/runners/system/tester_test.go @@ -618,7 +618,7 @@ func TestExpectedDatasets(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { r := &tester{} - got, err := r.expectedDatasets(c.scenario, nil) + got, err := r.expectedDatasets(c.scenario) require.NoError(t, err) assert.Equal(t, c.expected, got) }) From 2fee08af73b00b410976b676db466c4ba091bc9c Mon Sep 17 00:00:00 2001 From: Tere Date: Fri, 10 Apr 2026 16:00:54 +0200 Subject: [PATCH 10/12] Enhance otelcol input handling in documentation and code. Updated AGENTS.md to clarify that `data_stream.dataset` should not include a `.otel` suffix, as Elastic Agent appends it during ingestion. Modified packagepolicy.go and tester.go to ensure consistent handling of dataset names, preventing double suffixes in Elasticsearch. Updated system tests to reflect these changes and improve clarity in test case titles. --- AGENTS.md | 1 + internal/kibana/packagepolicy.go | 5 +++ internal/testrunner/runners/system/tester.go | 31 ++++++++++--------- .../testrunner/runners/system/tester_test.go | 30 +++++++++++------- 4 files changed, 41 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 966b4c0526..1cdd73c9b7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,3 +69,4 @@ elastic-package uses the objects-based Fleet API (`PackagePolicy`) — not the d - **Sibling stream disabling**: Fleet auto-enables all streams for an enabled input unless they are explicitly listed with `enabled: false`. Always send `{enabled: false}` for every sibling data stream sharing the same input type within the same policy template. - **Policy template scoping**: When a policy template declares a `data_streams` list, only include data streams from that list as siblings. Use `packages.FilterDatastreamsForPolicyTemplate(datastreams, policyTemplate)` on the result of `packages.ReadAllDataStreamManifests(packageRoot)` to get the correct set. - **Variable format**: the objects-based API expects raw values, not `{"type": ..., "value": ...}` wrappers. `Vars.ToMapStr()` extracts raw values via `val.Value.Value()`. +- **otelcol input packages**: `data_stream.dataset` is the **base** dataset name (without a `.otel` suffix). Elastic Agent appends `.otel` at ingest; Fleet does not add it in the policy payload. System tests in elastic-package mirror that append when resolving data stream names and expected document datasets—do not put `.otel` in the override unless you intentionally want a `*.otel.otel` dataset in Elasticsearch. diff --git a/internal/kibana/packagepolicy.go b/internal/kibana/packagepolicy.go index 9bfd4aee0a..b908e5d4d6 100644 --- a/internal/kibana/packagepolicy.go +++ b/internal/kibana/packagepolicy.go @@ -165,6 +165,8 @@ func BuildInputPackagePolicy( vars := SetKibanaVariables(policyTemplate.Vars, varValues) ensureDatasetVar(vars, policyTemplate, varValues) + // For otelcol, data_stream.dataset is the base name only (do not include a ".otel" suffix). + // Elastic Agent appends ".otel" at ingest; Fleet and this builder do not. if policyTemplate.Input == "otelcol" { ensureUseAPMVar(vars, varValues) } @@ -221,6 +223,9 @@ func datasetKey(pkgName string, ds packages.DataStreamManifest) string { // 2. manifest default already parsed into vars — promoted to fromUser=true // 3. explicit default in the policy-template var definitions // 4. policy template name as a final fallback +// +// For policy templates whose input is otelcol, callers should treat the stored value as the +// base dataset (see BuildInputPackagePolicy); the ".otel" routing suffix is applied by Elastic Agent. func ensureDatasetVar(vars Vars, policyTemplate packages.PolicyTemplate, varValues common.MapStr) { if raw, err := varValues.GetValue("data_stream.dataset"); err == nil { var val packages.VarValue diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index c1dafc88b0..9472fa5bc2 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -1510,19 +1510,24 @@ func filterOtelAPMRollupDataStreams(streams []discoveredDataStream) []discovered }) } -// BuildDataStreamName builds the expected data stream name that is installed in Elasticsearch -// when the package data stream is added to the policy. +// appendOtelcolAgentDatasetSuffix returns baseDataset + ".otel". Elastic Agent appends this +// routing suffix to the Fleet data_stream.dataset value for input packages that use the +// otelcol input; Fleet and elastic-package policy builders do not add it. System tests model +// the agent here when building Elasticsearch data stream names and expected document datasets. +// Configure data_stream.dataset as the base name only (without ".otel"); if the policy value +// already ends in ".otel", Elasticsearch will see a double suffix (...otel.otel). +func appendOtelcolAgentDatasetSuffix(baseDataset string) string { + return baseDataset + "." + otelSuffixDataset +} +// BuildDataStreamName builds the expected data stream name that is installed in Elasticsearch +// when the package data stream is added to the policy. For input packages with the otelcol +// input, the name includes the ".otel" suffix that Elastic Agent adds (see appendOtelcolAgentDatasetSuffix). func BuildDataStreamName(dsType, dsDataset, namespace string, policyTemplate packages.PolicyTemplate, packageType string) string { dataset := dsDataset - - // Input packages using the otel collector input require to add a specific dataset suffix if packageType == "input" && policyTemplate.Input == otelCollectorInputName { - if !strings.HasSuffix(dataset, "."+otelSuffixDataset) { - dataset = dataset + "." + otelSuffixDataset - } + dataset = appendOtelcolAgentDatasetSuffix(dataset) } - return fmt.Sprintf("%s-%s-%s", dsType, dataset, namespace) } @@ -2007,13 +2012,9 @@ func (r *tester) expectedDatasets(scenario *scenarioTest) ([]string, error) { if len(expectedDatasets) == 0 { // get dataset directly from package policy added when preparing the scenario expectedDataset := scenario.dataStreamDataset - if scenario.policyTemplate.Input == otelCollectorInputName { - // Input packages whose input is `otelcol` must add the `.otel` suffix. - // The suffix is conditional to avoid double-appending when the stored dataset - // already ends in `.otel` (user-configured via data_stream.dataset). - if !strings.HasSuffix(expectedDataset, "."+otelSuffixDataset) { - expectedDataset += "." + otelSuffixDataset - } + if scenario.policyTemplate.Input == otelCollectorInputName && + r.pkgManifest != nil && r.pkgManifest.Type == "input" { + expectedDataset = appendOtelcolAgentDatasetSuffix(expectedDataset) } expectedDatasets = []string{expectedDataset} } diff --git a/internal/testrunner/runners/system/tester_test.go b/internal/testrunner/runners/system/tester_test.go index 044fe357dc..58b72c7950 100644 --- a/internal/testrunner/runners/system/tester_test.go +++ b/internal/testrunner/runners/system/tester_test.go @@ -532,13 +532,13 @@ func TestBuildDataStreamName(t *testing.T) { expected: "logs-httpcheck.otel-default", }, { - title: "otelcol input: no double-suffix when dataset already ends in .otel", + title: "otelcol input: policy dataset already ending in .otel yields ...otel.otel (agent still appends)", dsType: "logs", dsDataset: "custom.otel", namespace: "default", policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, packageType: "input", - expected: "logs-custom.otel-default", + expected: "logs-custom.otel.otel-default", }, { title: "otelcol input on integration package type: no suffix added", @@ -561,9 +561,10 @@ func TestBuildDataStreamName(t *testing.T) { func TestExpectedDatasets(t *testing.T) { cases := []struct { - title string - scenario *scenarioTest - expected []string + title string + packageType string // tester.pkgManifest.Type; empty leaves pkgManifest nil (no otel append) + scenario *scenarioTest + expected []string }{ { title: "non-otelcol package: dataset returned as-is", @@ -574,7 +575,8 @@ func TestExpectedDatasets(t *testing.T) { expected: []string{"nginx.access"}, }, { - title: "otelcol input: .otel suffix appended", + title: "otelcol input package: .otel suffix appended like Elastic Agent", + packageType: "input", scenario: &scenarioTest{ dataStreamDataset: "httpcheck", policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, @@ -582,15 +584,17 @@ func TestExpectedDatasets(t *testing.T) { expected: []string{"httpcheck.otel"}, }, { - title: "otelcol input with user dataset already ending in .otel: no double-suffix", + title: "otelcol input package: policy value already ending in .otel still gets agent suffix", + packageType: "input", scenario: &scenarioTest{ dataStreamDataset: "generic.otel", policyTemplate: packages.PolicyTemplate{Input: otelCollectorInputName}, }, - expected: []string{"generic.otel"}, + expected: []string{"generic.otel.otel"}, }, { - title: "otelcol dynamic_signal_types: uses stored dataset, not policyTemplate.Name", + title: "otelcol dynamic_signal_types: uses stored dataset, not policyTemplate.Name", + packageType: "input", scenario: &scenarioTest{ dataStreamDataset: "sqlserverreceiver", policyTemplate: packages.PolicyTemplate{ @@ -602,7 +606,8 @@ func TestExpectedDatasets(t *testing.T) { expected: []string{"sqlserverreceiver.otel"}, }, { - title: "otelcol dynamic_signal_types with user dataset already ending in .otel: no double-suffix", + title: "otelcol dynamic_signal_types: policy value ending in .otel gets agent suffix", + packageType: "input", scenario: &scenarioTest{ dataStreamDataset: "generic.otel", policyTemplate: packages.PolicyTemplate{ @@ -611,13 +616,16 @@ func TestExpectedDatasets(t *testing.T) { DynamicSignalTypes: true, }, }, - expected: []string{"generic.otel"}, + expected: []string{"generic.otel.otel"}, }, } for _, c := range cases { t.Run(c.title, func(t *testing.T) { r := &tester{} + if c.packageType != "" { + r.pkgManifest = &packages.PackageManifest{Type: c.packageType} + } got, err := r.expectedDatasets(c.scenario) require.NoError(t, err) assert.Equal(t, c.expected, got) From b6dab53f39096c267236288cafa1805d1dfca327 Mon Sep 17 00:00:00 2001 From: Tere Date: Fri, 10 Apr 2026 16:18:53 +0200 Subject: [PATCH 11/12] Update MySQL and SQL Server input configurations and sample events. Changed default dataset names to 'custom.default' in MySQL input manifest and sample events. Updated timestamps and port numbers in sample event logs and metrics for both MySQL and SQL Server. Adjusted expected dataset names in test policies for consistency. Removed outdated skip reasons from test configurations. --- .../_dev/test/policy/test-default.expected | 4 +- .../parallel/mysql_input_otel/manifest.yml | 4 +- .../mysql_input_otel/sample_event_logs.json | 34 ++++----- .../sample_event_metrics.json | 38 ++++++---- .../test/system/test-all-signals-config.yml | 3 - .../_dev/test/system/test-default-config.yml | 3 - .../_dev/test/system/test-logs-config.yml | 3 - .../sql_server_input_otel/docs/README.md | 20 ++--- .../sql_server_input_otel/sample_event.json | 20 ++--- .../sample_event_logs.json | 76 +++++++++---------- .../sample_event_metrics.json | 20 ++--- .../test-default-custom-dataset.expected | 3 +- .../policy/test-default-custom-dataset.yml | 6 +- .../test-default-disable-use-apm.expected | 1 + .../policy/test-default-disable-use-apm.yml | 3 - .../test/policy/test-default-use-apm.expected | 1 + .../_dev/test/policy/test-default-use-apm.yml | 3 - .../test/policy/test-default-vars.expected | 1 + .../_dev/test/policy/test-default-vars.yml | 3 - .../_dev/test/policy/test-default.expected | 1 + .../_dev/test/policy/test-default.yml | 3 - .../_dev/test/system/test-default-config.yml | 3 - .../test-default-custom-dataset-config.yml | 11 +++ 23 files changed, 127 insertions(+), 137 deletions(-) create mode 100644 test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-custom-dataset-config.yml diff --git a/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected b/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected index 3840154aaf..c84acbf7ea 100644 --- a/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected +++ b/test/packages/parallel/mysql_input_otel/_dev/test/policy/test-default.expected @@ -33,13 +33,13 @@ processors: - context: log statements: - set(attributes["data_stream.type"], "logs") - - set(attributes["data_stream.dataset"], "mysql_input_otel.generic") + - set(attributes["data_stream.dataset"], "custom.default") - set(attributes["data_stream.namespace"], "ep") metric_statements: - context: datapoint statements: - set(attributes["data_stream.type"], "metrics") - - set(attributes["data_stream.dataset"], "mysql_input_otel.generic") + - set(attributes["data_stream.dataset"], "custom.default") - set(attributes["data_stream.namespace"], "ep") receivers: mysql/componentid-0: diff --git a/test/packages/parallel/mysql_input_otel/manifest.yml b/test/packages/parallel/mysql_input_otel/manifest.yml index 6f082ba24b..b1e4889cd0 100644 --- a/test/packages/parallel/mysql_input_otel/manifest.yml +++ b/test/packages/parallel/mysql_input_otel/manifest.yml @@ -253,9 +253,7 @@ policy_templates: - name: data_stream.dataset type: text title: Dataset name - description: | - Dataset to write data to. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). - default: mysql_input_otel.generic + default: custom.default required: true show_user: true owner: diff --git a/test/packages/parallel/mysql_input_otel/sample_event_logs.json b/test/packages/parallel/mysql_input_otel/sample_event_logs.json index a1a7ea1c2e..9dc559799b 100644 --- a/test/packages/parallel/mysql_input_otel/sample_event_logs.json +++ b/test/packages/parallel/mysql_input_otel/sample_event_logs.json @@ -1,9 +1,9 @@ { - "@timestamp": "2026-04-08T13:54:17.208Z", + "@timestamp": "2026-04-10T13:54:38.937Z", "attributes": { "client": { "address": "172.29.0.2", - "port": 39480 + "port": 53476 }, "db": { "namespace": "", @@ -25,14 +25,14 @@ "threads": { "processlist_command": "Query", "processlist_state": "Opening tables", - "thread_id": 52 + "thread_id": 51 }, "wait_type": "other" }, "network": { "peer": { "address": "172.29.0.2", - "port": 39480 + "port": 53476 } }, "user": { @@ -41,11 +41,11 @@ }, "client": { "address": "172.29.0.2", - "port": 39480 + "port": 53476 }, "data_stream": { - "dataset": "mysql_input_otel.generic.otel", - "namespace": "53297", + "dataset": "custom.default.otel", + "namespace": "74861", "type": "logs" }, "db": { @@ -59,12 +59,12 @@ }, "event": { "agent_id_status": "missing", - "dataset": "mysql_input_otel.generic.otel", - "ingested": "2026-04-08T13:54:27Z" + "dataset": "custom.default.otel", + "ingested": "2026-04-10T13:54:48Z" }, "event_name": "db.server.query_sample", "host": { - "name": "elastic-agent-23953", + "name": "elastic-agent-31189", "os": { "platform": "linux" } @@ -80,14 +80,14 @@ "threads": { "processlist_command": "Query", "processlist_state": "Opening tables", - "thread_id": 52 + "thread_id": 51 }, "wait_type": "other" }, "network": { "peer": { "address": "172.29.0.2", - "port": 39480 + "port": 53476 } }, "observed_timestamp": "1970-01-01T00:00:00.000Z", @@ -97,7 +97,7 @@ "resource": { "attributes": { "host": { - "name": "elastic-agent-23953" + "name": "elastic-agent-31189" }, "os": { "type": "linux" @@ -110,13 +110,13 @@ "version": "9.4.0" }, "span": { - "id": "8439d48bb0ce6738" + "id": "af7ceb14167b2ddc" }, - "span_id": "8439d48bb0ce6738", + "span_id": "af7ceb14167b2ddc", "trace": { - "id": "2c3069c1a55670bee4d912639a0e184c" + "id": "be4b906a8849095943cf0e857680bc23" }, - "trace_id": "2c3069c1a55670bee4d912639a0e184c", + "trace_id": "be4b906a8849095943cf0e857680bc23", "user": { "name": "root" } diff --git a/test/packages/parallel/mysql_input_otel/sample_event_metrics.json b/test/packages/parallel/mysql_input_otel/sample_event_metrics.json index ed22eb4495..487f96c94a 100644 --- a/test/packages/parallel/mysql_input_otel/sample_event_metrics.json +++ b/test/packages/parallel/mysql_input_otel/sample_event_metrics.json @@ -1,35 +1,44 @@ { - "@timestamp": "2026-04-08T13:54:17.208Z", - "_metric_names_hash": "800d765540031faf", + "@timestamp": "2026-04-10T13:54:38.937Z", + "_metric_names_hash": "9541e212302c9863", "attributes": { - "status": "overflow" + "kind": "sent" }, "data_stream": { - "dataset": "mysql_input_otel.generic.otel", - "namespace": "53297", + "dataset": "custom.default.otel", + "namespace": "74861", "type": "metrics" }, "event": { "agent_id_status": "missing", - "dataset": "mysql_input_otel.generic.otel", - "ingested": "2026-04-08T13:54:27Z" + "dataset": "custom.default.otel", + "ingested": "2026-04-10T13:54:49Z" }, "host": { - "name": "elastic-agent-23953", + "name": "elastic-agent-31189", "os": { "platform": "linux" } }, + "kind": "sent", "metrics": { "mysql": { - "table_open_cache": 0 + "client": { + "network": { + "io": 17901 + } + } } }, "mysql": { + "client": { + "network": { + "io": 17901 + } + }, "instance": { "endpoint": "svc-mysql_input_otel:3306" - }, - "table_open_cache": 0 + } }, "os": { "type": "linux" @@ -37,7 +46,7 @@ "resource": { "attributes": { "host": { - "name": "elastic-agent-23953" + "name": "elastic-agent-31189" }, "mysql": { "instance": { @@ -54,7 +63,6 @@ "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver", "version": "9.4.0" }, - "start_timestamp": "2026-04-08T13:54:16.199Z", - "status": "overflow", - "unit": "1" + "start_timestamp": "2026-04-10T13:54:37.923Z", + "unit": "By" } diff --git a/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-all-signals-config.yml b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-all-signals-config.yml index e6b5b43936..29b58b6235 100644 --- a/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-all-signals-config.yml +++ b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-all-signals-config.yml @@ -1,6 +1,3 @@ -skip: - reason: Broken after change in Kibana, to be fixed after https://github.com/elastic/kibana/pull/262000 - link: https://github.com/elastic/elastic-package/pull/3433 service: sql_server_input_otel signal_types: - logs diff --git a/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-default-config.yml b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-default-config.yml index 676c6c8b50..05a0e22b2a 100644 --- a/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-default-config.yml +++ b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-default-config.yml @@ -1,6 +1,3 @@ -skip: - reason: Broken after change in Kibana, to be fixed after https://github.com/elastic/kibana/pull/262000 - link: https://github.com/elastic/elastic-package/pull/3433 service: sql_server_input_otel vars: server: "{{Hostname}}" diff --git a/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-logs-config.yml b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-logs-config.yml index ea922cdf1e..8e4287833b 100644 --- a/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-logs-config.yml +++ b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-logs-config.yml @@ -1,6 +1,3 @@ -skip: - reason: Broken after change in Kibana, to be fixed after https://github.com/elastic/kibana/pull/262000 - link: https://github.com/elastic/elastic-package/pull/3433 service: sql_server_input_otel signal_types: - logs diff --git a/test/packages/parallel/sql_server_input_otel/docs/README.md b/test/packages/parallel/sql_server_input_otel/docs/README.md index 53d787762b..212d921b18 100644 --- a/test/packages/parallel/sql_server_input_otel/docs/README.md +++ b/test/packages/parallel/sql_server_input_otel/docs/README.md @@ -57,25 +57,25 @@ An example event looks as following: ```json { - "@timestamp": "2026-04-08T13:51:30.682Z", + "@timestamp": "2026-04-10T13:49:55.739Z", "_metric_names_hash": "7ee276998c966647", "attributes": { "performance_counter": { - "object_name": "SQLServer:Buffer Manager" + "object_name": "SQLServer:Buffer Node" } }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "24063", + "namespace": "40722", "type": "metrics" }, "event": { "agent_id_status": "missing", "dataset": "sqlserverreceiver.otel", - "ingested": "2026-04-08T13:51:40Z" + "ingested": "2026-04-10T13:50:05Z" }, "host": { - "name": "elastic-agent-75226", + "name": "elastic-agent-70881", "os": { "platform": "linux" } @@ -83,7 +83,7 @@ An example event looks as following: "metrics": { "sqlserver": { "page": { - "life_expectancy": 16 + "life_expectancy": 21 } } }, @@ -91,12 +91,12 @@ An example event looks as following: "type": "linux" }, "performance_counter": { - "object_name": "SQLServer:Buffer Manager" + "object_name": "SQLServer:Buffer Node" }, "resource": { "attributes": { "host": { - "name": "elastic-agent-75226" + "name": "elastic-agent-70881" }, "os": { "type": "linux" @@ -123,10 +123,10 @@ An example event looks as following: }, "sqlserver": { "page": { - "life_expectancy": 16 + "life_expectancy": 21 } }, - "start_timestamp": "2026-04-08T13:51:29.559Z", + "start_timestamp": "2026-04-10T13:49:54.565Z", "unit": "s" } ``` diff --git a/test/packages/parallel/sql_server_input_otel/sample_event.json b/test/packages/parallel/sql_server_input_otel/sample_event.json index b03f7efec9..76ea27f560 100644 --- a/test/packages/parallel/sql_server_input_otel/sample_event.json +++ b/test/packages/parallel/sql_server_input_otel/sample_event.json @@ -1,23 +1,23 @@ { - "@timestamp": "2026-04-08T13:51:30.682Z", + "@timestamp": "2026-04-10T13:49:55.739Z", "_metric_names_hash": "7ee276998c966647", "attributes": { "performance_counter": { - "object_name": "SQLServer:Buffer Manager" + "object_name": "SQLServer:Buffer Node" } }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "24063", + "namespace": "40722", "type": "metrics" }, "event": { "agent_id_status": "missing", "dataset": "sqlserverreceiver.otel", - "ingested": "2026-04-08T13:51:40Z" + "ingested": "2026-04-10T13:50:05Z" }, "host": { - "name": "elastic-agent-75226", + "name": "elastic-agent-70881", "os": { "platform": "linux" } @@ -25,7 +25,7 @@ "metrics": { "sqlserver": { "page": { - "life_expectancy": 16 + "life_expectancy": 21 } } }, @@ -33,12 +33,12 @@ "type": "linux" }, "performance_counter": { - "object_name": "SQLServer:Buffer Manager" + "object_name": "SQLServer:Buffer Node" }, "resource": { "attributes": { "host": { - "name": "elastic-agent-75226" + "name": "elastic-agent-70881" }, "os": { "type": "linux" @@ -65,9 +65,9 @@ }, "sqlserver": { "page": { - "life_expectancy": 16 + "life_expectancy": 21 } }, - "start_timestamp": "2026-04-08T13:51:29.559Z", + "start_timestamp": "2026-04-10T13:49:54.565Z", "unit": "s" } diff --git a/test/packages/parallel/sql_server_input_otel/sample_event_logs.json b/test/packages/parallel/sql_server_input_otel/sample_event_logs.json index db1ebd67ba..cee38f5849 100644 --- a/test/packages/parallel/sql_server_input_otel/sample_event_logs.json +++ b/test/packages/parallel/sql_server_input_otel/sample_event_logs.json @@ -1,15 +1,12 @@ { - "@timestamp": "2026-04-08T13:50:11.062Z", + "@timestamp": "2026-04-10T13:47:44.483Z", "attributes": { "client": { - "address": "elastic-agent-71667", - "port": 32812 + "address": "elastic-agent-15653", + "port": 49412 }, "db": { "namespace": "master", - "query": { - "text": "WITH PerfCounters SELECT DISTINCT RTRIM ( spi. object_name ) object_name, RTRIM ( spi. counter_name ) counter_name, RTRIM ( spi. instance_name ), CAST ( spi. cntr_value ), spi. cntr_type FROM sys.dm_os_performance_counters WHERE counter_name IN ( ? ) OR ( spi. object_name LIKE ? OR spi. object_name LIKE ? OR spi. object_name LIKE ? ) OR ( spi. instance_name IN ( ? ) AND spi. counter_name IN ( ? ) ) ) INSERT INTO @PCounters SELECT * FROM PerfCounters" - }, "system": { "name": "microsoft.sql_server" } @@ -17,56 +14,53 @@ "network": { "peer": { "address": "172.29.0.2", - "port": 32812 + "port": 49412 } }, "sqlserver": { "blocking_session_id": 0, - "command": "INSERT", + "command": "SELECT", "context_info": "", - "cpu_time": 0.052, - "deadlock_priority": -10, + "cpu_time": 0.061, + "deadlock_priority": 0, "estimated_completion_time": 0, "lock_timeout": -0.001, - "logical_reads": 607, + "logical_reads": 669, "open_transaction_count": 0, "percent_complete": 0, "procedure_id": "0", "procedure_name": "", - "query_hash": "6df033ef55623a4a", - "query_plan_hash": "49c16c014d7e4505", - "query_start": "2026-04-08T13:50:10.997+00:00", - "reads": 40, + "query_hash": "30447d7d5ef3066a", + "query_plan_hash": "0c97dc34ff850504", + "query_start": "2026-04-10T13:47:44.407+00:00", + "reads": 104, "request_status": "running", - "row_count": 1, + "row_count": 0, "session_id": 51, "session_status": "running", - "total_elapsed_time": 0.053, - "transaction_id": 0, + "total_elapsed_time": 0.063, + "transaction_id": 1881, "transaction_isolation_level": 2, "wait_resource": "", "wait_time": 0, "wait_type": "", - "writes": 19 + "writes": 0 }, "user": { "name": "sa" } }, "client": { - "address": "elastic-agent-71667", - "port": 32812 + "address": "elastic-agent-15653", + "port": 49412 }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "78202", + "namespace": "44690", "type": "logs" }, "db": { "namespace": "master", - "query": { - "text": "WITH PerfCounters SELECT DISTINCT RTRIM ( spi. object_name ) object_name, RTRIM ( spi. counter_name ) counter_name, RTRIM ( spi. instance_name ), CAST ( spi. cntr_value ), spi. cntr_type FROM sys.dm_os_performance_counters WHERE counter_name IN ( ? ) OR ( spi. object_name LIKE ? OR spi. object_name LIKE ? OR spi. object_name LIKE ? ) OR ( spi. instance_name IN ( ? ) AND spi. counter_name IN ( ? ) ) ) INSERT INTO @PCounters SELECT * FROM PerfCounters" - }, "system": { "name": "microsoft.sql_server" } @@ -74,11 +68,11 @@ "event": { "agent_id_status": "missing", "dataset": "sqlserverreceiver.otel", - "ingested": "2026-04-08T13:50:21Z" + "ingested": "2026-04-10T13:47:54Z" }, "event_name": "db.server.query_sample", "host": { - "name": "elastic-agent-71667", + "name": "elastic-agent-15653", "os": { "platform": "linux" } @@ -86,7 +80,7 @@ "network": { "peer": { "address": "172.29.0.2", - "port": 32812 + "port": 49412 } }, "observed_timestamp": "1970-01-01T00:00:00.000Z", @@ -96,7 +90,7 @@ "resource": { "attributes": { "host": { - "name": "elastic-agent-71667" + "name": "elastic-agent-15653" }, "os": { "type": "linux" @@ -123,32 +117,32 @@ }, "sqlserver": { "blocking_session_id": 0, - "command": "INSERT", + "command": "SELECT", "context_info": "", - "cpu_time": 0.052, - "deadlock_priority": -10, + "cpu_time": 0.061, + "deadlock_priority": 0, "estimated_completion_time": 0, "lock_timeout": -0.001, - "logical_reads": 607, + "logical_reads": 669, "open_transaction_count": 0, "percent_complete": 0, "procedure_id": "0", "procedure_name": "", - "query_hash": "6df033ef55623a4a", - "query_plan_hash": "49c16c014d7e4505", - "query_start": "2026-04-08T13:50:10.997+00:00", - "reads": 40, + "query_hash": "30447d7d5ef3066a", + "query_plan_hash": "0c97dc34ff850504", + "query_start": "2026-04-10T13:47:44.407+00:00", + "reads": 104, "request_status": "running", - "row_count": 1, + "row_count": 0, "session_id": 51, "session_status": "running", - "total_elapsed_time": 0.053, - "transaction_id": 0, + "total_elapsed_time": 0.063, + "transaction_id": 1881, "transaction_isolation_level": 2, "wait_resource": "", "wait_time": 0, "wait_type": "", - "writes": 19 + "writes": 0 }, "user": { "name": "sa" diff --git a/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json b/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json index dbf4134d14..4f38e23c64 100644 --- a/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json +++ b/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json @@ -1,23 +1,23 @@ { - "@timestamp": "2026-04-08T13:50:11.123Z", + "@timestamp": "2026-04-10T13:47:44.574Z", "_metric_names_hash": "7ee276998c966647", "attributes": { "performance_counter": { - "object_name": "SQLServer:Buffer Manager" + "object_name": "SQLServer:Buffer Node" } }, "data_stream": { "dataset": "sqlserverreceiver.otel", - "namespace": "78202", + "namespace": "44690", "type": "metrics" }, "event": { "agent_id_status": "missing", "dataset": "sqlserverreceiver.otel", - "ingested": "2026-04-08T13:50:21Z" + "ingested": "2026-04-10T13:47:54Z" }, "host": { - "name": "elastic-agent-71667", + "name": "elastic-agent-15653", "os": { "platform": "linux" } @@ -25,7 +25,7 @@ "metrics": { "sqlserver": { "page": { - "life_expectancy": 17 + "life_expectancy": 20 } } }, @@ -33,12 +33,12 @@ "type": "linux" }, "performance_counter": { - "object_name": "SQLServer:Buffer Manager" + "object_name": "SQLServer:Buffer Node" }, "resource": { "attributes": { "host": { - "name": "elastic-agent-71667" + "name": "elastic-agent-15653" }, "os": { "type": "linux" @@ -65,9 +65,9 @@ }, "sqlserver": { "page": { - "life_expectancy": 17 + "life_expectancy": 20 } }, - "start_timestamp": "2026-04-08T13:50:09.970Z", + "start_timestamp": "2026-04-10T13:47:43.383Z", "unit": "s" } diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected index cddb652cb4..6fad817d9e 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected @@ -42,11 +42,12 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") - - set(attributes["data_stream.dataset"], "zipkin.custom") + - set(attributes["data_stream.dataset"], "generic") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: - set(attributes["data_stream.type"], "logs") + - set(attributes["data_stream.dataset"], "generic") - set(attributes["data_stream.namespace"], "ep") receivers: zipkin/componentid-0: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.yml b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.yml index df2ac89b94..63a95c4ca8 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.yml +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.yml @@ -1,7 +1,5 @@ -skip: - reason: Broken after change in Kibana, to be fixed after https://github.com/elastic/kibana/pull/262000 - link: https://github.com/elastic/elastic-package/pull/3433 vars: endpoint: "0.0.0.0:9411" parse_string_tags: true - data_stream.dataset: "zipkin.custom" + # Base dataset only; Elastic Agent appends ".otel" at ingest. + data_stream.dataset: "generic" diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected index 9bfab820c3..6ee40722de 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.expected @@ -35,6 +35,7 @@ processors: - context: spanevent statements: - set(attributes["data_stream.type"], "logs") + - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") receivers: zipkin/componentid-0: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.yml b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.yml index e5f9203715..70e101c31d 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.yml +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-disable-use-apm.yml @@ -1,6 +1,3 @@ -skip: - reason: Broken after change in Kibana, to be fixed after https://github.com/elastic/kibana/pull/262000 - link: https://github.com/elastic/elastic-package/pull/3433 vars: endpoint: "0.0.0.0:9411" parse_string_tags: true diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected index 005a7c72b6..d68ccb29d6 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.expected @@ -47,6 +47,7 @@ processors: - context: spanevent statements: - set(attributes["data_stream.type"], "logs") + - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") receivers: zipkin/componentid-0: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.yml b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.yml index ded0e207b0..7b383dfd44 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.yml +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-use-apm.yml @@ -1,6 +1,3 @@ -skip: - reason: Broken after change in Kibana, to be fixed after https://github.com/elastic/kibana/pull/262000 - link: https://github.com/elastic/elastic-package/pull/3433 vars: endpoint: "0.0.0.0:9411" parse_string_tags: true diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected index c23cff1583..6b934211df 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.expected @@ -47,6 +47,7 @@ processors: - context: spanevent statements: - set(attributes["data_stream.type"], "logs") + - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") receivers: zipkin/componentid-0: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.yml b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.yml index 32a3eb1ff0..72b4ca320d 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.yml +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-vars.yml @@ -1,4 +1 @@ -skip: - reason: Broken after change in Kibana, to be fixed after https://github.com/elastic/kibana/pull/262000 - link: https://github.com/elastic/elastic-package/pull/3433 vars: ~ diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected index 005a7c72b6..d68ccb29d6 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.expected @@ -47,6 +47,7 @@ processors: - context: spanevent statements: - set(attributes["data_stream.type"], "logs") + - set(attributes["data_stream.dataset"], "zipkinreceiver") - set(attributes["data_stream.namespace"], "ep") receivers: zipkin/componentid-0: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.yml b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.yml index 5a74a56acd..5893fe8d4f 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.yml +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default.yml @@ -1,6 +1,3 @@ -skip: - reason: Broken after change in Kibana, to be fixed after https://github.com/elastic/kibana/pull/262000 - link: https://github.com/elastic/elastic-package/pull/3433 vars: endpoint: "0.0.0.0:9411" parse_string_tags: true diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-config.yml b/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-config.yml index a504b2be95..6b819b643e 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-config.yml +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-config.yml @@ -1,6 +1,3 @@ -skip: - reason: Broken after change in Kibana, to be fixed after https://github.com/elastic/kibana/pull/262000 - link: https://github.com/elastic/elastic-package/pull/3433 service: backend vars: endpoint: "0.0.0.0:9411" diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-custom-dataset-config.yml b/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-custom-dataset-config.yml new file mode 100644 index 0000000000..12e4baf1a3 --- /dev/null +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-custom-dataset-config.yml @@ -0,0 +1,11 @@ +service: backend +vars: + endpoint: "0.0.0.0:9411" + parse_string_tags: false + use_apm: true + # Base dataset only; Elastic Agent appends ".otel" at ingest. + data_stream.dataset: "generic" +assert: + min_count: 20 +# required to increase waiting time to detect all data streams (specially logs and traces) +wait_for_dynamic_streams_stable: 90s From 05a69e52aa9b8dce6d815ed7dbaf4337b3afee86 Mon Sep 17 00:00:00 2001 From: Tere Date: Thu, 16 Apr 2026 11:55:41 +0200 Subject: [PATCH 12/12] Update zipking after fix merged --- .../test-default-custom-dataset.expected | 2 +- .../policy/test-default-custom-dataset.yml | 3 +- .../test-default-custom-dataset-config.yml | 2 +- .../zipkin_input_otel/sample_event.json | 43 +++++++++---------- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected index dd7397d83c..01f9bb86e1 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.expected @@ -42,7 +42,7 @@ processors: - context: span statements: - set(attributes["data_stream.type"], "traces") - - set(attributes["data_stream.dataset"], "generic") + - set(attributes["data_stream.dataset"], "zipkin.custom") - set(attributes["data_stream.namespace"], "ep") - context: spanevent statements: diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.yml b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.yml index 63a95c4ca8..90f01adbef 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.yml +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/policy/test-default-custom-dataset.yml @@ -1,5 +1,4 @@ vars: endpoint: "0.0.0.0:9411" parse_string_tags: true - # Base dataset only; Elastic Agent appends ".otel" at ingest. - data_stream.dataset: "generic" + data_stream.dataset: "zipkin.custom" diff --git a/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-custom-dataset-config.yml b/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-custom-dataset-config.yml index 12e4baf1a3..be1ac6626f 100644 --- a/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-custom-dataset-config.yml +++ b/test/packages/parallel/zipkin_input_otel/_dev/test/system/test-default-custom-dataset-config.yml @@ -4,7 +4,7 @@ vars: parse_string_tags: false use_apm: true # Base dataset only; Elastic Agent appends ".otel" at ingest. - data_stream.dataset: "generic" + data_stream.dataset: "zipkin.custom" assert: min_count: 20 # required to increase waiting time to detect all data streams (specially logs and traces) diff --git a/test/packages/parallel/zipkin_input_otel/sample_event.json b/test/packages/parallel/zipkin_input_otel/sample_event.json index 7dcbea40e1..7ed2f616cc 100644 --- a/test/packages/parallel/zipkin_input_otel/sample_event.json +++ b/test/packages/parallel/zipkin_input_otel/sample_event.json @@ -1,5 +1,5 @@ { - "@timestamp": "2026-03-05T13:05:14.439Z", + "@timestamp": "2026-04-16T09:52:22.084Z", "agent": { "name": "otlp", "version": "unknown" @@ -15,27 +15,24 @@ }, "net": { "host": { - "ip": "172.28.0.2" + "ip": "172.30.0.2" }, "peer": { - "ip": "172.28.0.3", - "port": 39034 + "ip": "172.30.0.3", + "port": 54920 } }, "processor": { "event": "transaction" }, - "span": { - "id": "3d3f1f56d1c91af8" - }, "timestamp": { - "us": 1772715914439970 + "us": 1776333142084949 }, "transaction": { "duration": { - "us": 2328 + "us": 855 }, - "id": "3d3f1f56d1c91af8", + "id": "1446bb067c058040", "name": "GET /api", "representative_count": 1, "result": "Success", @@ -46,14 +43,14 @@ }, "data_stream": { "dataset": "zipkinreceiver.otel", - "namespace": "50659", + "namespace": "33971", "type": "traces" }, - "duration": 2328000, + "duration": 855000, "event": { "agent_id_status": "missing", "dataset": "zipkinreceiver.otel", - "ingested": "2026-03-05T13:05:24Z", + "ingested": "2026-04-16T09:52:32Z", "outcome": "success", "success_count": 1 }, @@ -65,11 +62,11 @@ "name": "GET /api", "net": { "host": { - "ip": "172.28.0.2" + "ip": "172.30.0.2" }, "peer": { - "ip": "172.28.0.3", - "port": 39034 + "ip": "172.30.0.3", + "port": 54920 } }, "processor": { @@ -90,22 +87,22 @@ "name": "backend" }, "span": { - "id": "3d3f1f56d1c91af8", + "id": "1446bb067c058040", "name": "GET /api" }, - "span_id": "3d3f1f56d1c91af8", + "span_id": "1446bb067c058040", "timestamp": { - "us": 1772715914439970 + "us": 1776333142084949 }, "trace": { - "id": "00000000000000003d3f1f56d1c91af8" + "id": "00000000000000001446bb067c058040" }, - "trace_id": "00000000000000003d3f1f56d1c91af8", + "trace_id": "00000000000000001446bb067c058040", "transaction": { "duration": { - "us": 2328 + "us": 855 }, - "id": "3d3f1f56d1c91af8", + "id": "1446bb067c058040", "name": "GET /api", "representative_count": 1, "result": "Success",