diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index 8d61c7800..22a770d3e 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -37,6 +37,7 @@ func TestValidateFile(t *testing.T) { "good_input": {}, "good_input_otel": {}, "good_input_dynamic_signal_type": {}, + "good_input_profiles": {}, "good_input_template_paths": {}, "good_integration_template_paths": {}, "good_content": {}, @@ -390,6 +391,12 @@ func TestValidateFile(t *testing.T) { "field policy_templates.0.input: Must not be present", }, }, + "bad_input_profiles_non_otel": { + "manifest.yml", + []string{ + "field policy_templates.0.input: policy_templates.0.input must be one of the following: \"otelcol\"", + }, + }, "bad_input_dynamic_signal_types_non_otel": { "manifest.yml", []string{ diff --git a/compliance/features/basic.feature b/compliance/features/basic.feature index 10c34b4fe..eb3c43488 100644 --- a/compliance/features/basic.feature +++ b/compliance/features/basic.feature @@ -40,3 +40,9 @@ Feature: Basic package types support Given the "good_v3" package is installed And a policy is created with "good_v3" package, "1.1.0" version, "otel" template, "otelcol" input type and dataset "good_v3.otel_test" Then there is an index template "logs-good_v3.otel_test" with pattern "logs-good_v3.otel_test.otel-*" + + @3.6.0 + Scenario: OTel input package with profiles type can be installed + Given the "good_input_profiles" package is installed + And a policy is created with "good_input_profiles" package, "0.0.1" version, "profilingreceiver" template, "profilingreceiver" input, "otelcol" input type and dataset "spec.otel_input_test" + # We omit assertions here because 'profiles' OTel packages should not produce ES assets which is currently handled by an ES plugin. https://github.com/elastic/package-spec/pull/1092#discussion_r2857893043 \ No newline at end of file diff --git a/spec/changelog.yml b/spec/changelog.yml index 0a59c7c62..46c8a09ff 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -58,6 +58,12 @@ - description: Allow to force the format used in the policy API in system and policy tests. type: enhancement link: https://github.com/elastic/package-spec/pull/1103 + - description: Add support for auto_expand_replicas, max_result_window, and refresh_interval index settings in data stream templates. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1092 + - description: Add support for `profiles` policy template type in input packages + type: enhancement + link: https://github.com/elastic/package-spec/pull/1092 - version: 3.5.7 changes: - description: Allow _dev directory for content-only packages; use _dev/shared for development files (e.g. dashboard YML sources). diff --git a/spec/input/manifest.spec.yml b/spec/input/manifest.spec.yml index 896d0e1ec..91e9663b4 100644 --- a/spec/input/manifest.spec.yml +++ b/spec/input/manifest.spec.yml @@ -55,6 +55,7 @@ spec: - logs - synthetics - traces + - profiles examples: - metrics title: @@ -96,23 +97,36 @@ spec: allOf: # Either template_path or template_paths must be present (mutually exclusive) - oneOf: - - required: - - template_path - not: - required: - - template_paths - - required: + - required: + - template_path + not: + required: - template_paths - not: - required: - - template_path + - required: + - template_paths + not: + required: + - template_path # Either type is required, or dynamic_signal_types must be true - oneOf: + - allOf: - required: - - type - - properties: - dynamic_signal_types: - const: true + - type + # If type is profiles, can only use otelcol inputs + - if: + properties: + type: + const: profiles + then: + properties: + input: + enum: + - otelcol + - required: + - dynamic_signal_types + properties: + dynamic_signal_types: + const: true required: - name - title @@ -155,7 +169,7 @@ versions: - op: remove path: "/properties/policy_templates/items/properties/template_paths" # removes template_paths field for policy templates - op: remove - path: "/properties/policy_templates/items/allOf" # removes allOf constraint for template_path/template_paths and dynamic_signal_types/type + path: "/properties/policy_templates/items/allOf" # removes allOf constraint for template_path/template_paths, dynamic_signal_types/type, and profiles otelcol input - op: add path: "/properties/policy_templates/items/required" value: @@ -173,6 +187,8 @@ versions: path: "/properties/deprecated" # removes deprecated field for package - op: remove path: "/properties/policy_templates/items/properties/deprecated" # removes deprecated field for policy templates + - op: remove + path: "/properties/policy_templates/items/properties/type/enum/4" # removes 'profiles' as a valid policy template type, since it requires Kibana version >= 9.4.0 support # Reserve otelcol input name before 3.5.0. - before: 3.5.0 patch: diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index 7f0d4aa38..82ba27a8d 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -263,6 +263,12 @@ spec: $ref: "#/definitions/index_template_setting_index" number_of_shards: $ref: "#/definitions/index_template_setting_number_of_shards" + auto_expand_replicas: + $ref: "#/definitions/index_template_setting_auto_expand_replicas" + max_result_window: + $ref: "#/definitions/index_template_setting_max_result_window" + refresh_interval: + $ref: "#/definitions/index_template_setting_refresh_interval" mappings: description: Mappings section of index template type: object @@ -512,6 +518,32 @@ spec: type: integer minimum: 1 default: 1 + index_template_setting_auto_expand_replicas: + description: | + Auto-expand the number of replicas based on the number of data nodes in the cluster. + Set to a dash delimited lower and upper bound (e.g. 0-5) or use `all` for the upper bound (e.g. 0-all). Defaults to false (i.e. disabled). + type: string + examples: + - "0-1" + - "0-all" + - "false" + index_template_setting_max_result_window: + description: | + The maximum value of `from + size` for searches to this index. Defaults to 10000. + Search requests take heap memory and time proportional to `from + size` and this limits that memory. + type: integer + minimum: 1 + examples: + - 50000 + index_template_setting_refresh_interval: + description: | + How often to perform a refresh operation, which makes recent changes to the index visible to search. The value defaults to `1s` in Elastic Stack and `5s` in Serverless. In Serverless, `5s` is also the minimum value that can be set. + Can be set to `-1` to disable refresh. + type: string + examples: + - "10s" + - "1m" + - "-1" properties: dataset: description: Name of data set. diff --git a/test/packages/bad_input_profiles_non_otel/agent/input/input.yml.hbs b/test/packages/bad_input_profiles_non_otel/agent/input/input.yml.hbs new file mode 100644 index 000000000..f364d1ed8 --- /dev/null +++ b/test/packages/bad_input_profiles_non_otel/agent/input/input.yml.hbs @@ -0,0 +1,10 @@ +receivers: + profiling: +{{#if samples_per_second}} + samples_per_second: {{samples_per_second}} +{{/if}} + +service: + pipelines: + profiles: + receivers: [ profiling ] \ No newline at end of file diff --git a/test/packages/bad_input_profiles_non_otel/changelog.yml b/test/packages/bad_input_profiles_non_otel/changelog.yml new file mode 100644 index 000000000..6c5466283 --- /dev/null +++ b/test/packages/bad_input_profiles_non_otel/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - type: enhancement + description: Initial release for testing input policy templates with type `profiles` with a non otelcol input type. + link: https://github.com/elastic/package-spec/pull/1092 diff --git a/test/packages/bad_input_profiles_non_otel/docs/README.md b/test/packages/bad_input_profiles_non_otel/docs/README.md new file mode 100644 index 000000000..3c149b95e --- /dev/null +++ b/test/packages/bad_input_profiles_non_otel/docs/README.md @@ -0,0 +1 @@ +This is a template for the package README. diff --git a/test/packages/bad_input_profiles_non_otel/manifest.yml b/test/packages/bad_input_profiles_non_otel/manifest.yml new file mode 100644 index 000000000..ab1fe10e2 --- /dev/null +++ b/test/packages/bad_input_profiles_non_otel/manifest.yml @@ -0,0 +1,32 @@ +format_version: 3.6.0 +name: bad_input_profiles_non_otel +title: "Bad OpenTelemetry Profiling" +version: 0.0.1 +source: + license: "Elastic-2.0" +description: "Bad OpenTelemetry Profiling" +type: input +categories: + - custom + - observability + - opentelemetry +conditions: + kibana: + version: "^9.4.0" + elastic: + subscription: "basic" +policy_templates: + - name: profilingreceiver + type: profiles + title: OpenTelemetry Profiling receiver + description: Collect profiling data + input: logs + template_path: input.yml.hbs + vars: + - name: samples_per_second + type: integer + required: false + description: Defines the sampling frequency. +owner: + github: elastic/ingest-otel-data + type: elastic diff --git a/test/packages/good_input_profiles/agent/input/input.yml.hbs b/test/packages/good_input_profiles/agent/input/input.yml.hbs new file mode 100644 index 000000000..f364d1ed8 --- /dev/null +++ b/test/packages/good_input_profiles/agent/input/input.yml.hbs @@ -0,0 +1,10 @@ +receivers: + profiling: +{{#if samples_per_second}} + samples_per_second: {{samples_per_second}} +{{/if}} + +service: + pipelines: + profiles: + receivers: [ profiling ] \ No newline at end of file diff --git a/test/packages/good_input_profiles/changelog.yml b/test/packages/good_input_profiles/changelog.yml new file mode 100644 index 000000000..d70ea67cf --- /dev/null +++ b/test/packages/good_input_profiles/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - type: enhancement + description: Initial release for testing input policy templates with type `profiles`. + link: https://github.com/elastic/package-spec/pull/1092 diff --git a/test/packages/good_input_profiles/docs/README.md b/test/packages/good_input_profiles/docs/README.md new file mode 100644 index 000000000..3c149b95e --- /dev/null +++ b/test/packages/good_input_profiles/docs/README.md @@ -0,0 +1 @@ +This is a template for the package README. diff --git a/test/packages/good_input_profiles/manifest.yml b/test/packages/good_input_profiles/manifest.yml new file mode 100644 index 000000000..f64b09cc6 --- /dev/null +++ b/test/packages/good_input_profiles/manifest.yml @@ -0,0 +1,32 @@ +format_version: 3.6.0 +name: good_input_profiles +title: "OpenTelemetry Profiling" +version: 0.0.1 +source: + license: "Elastic-2.0" +description: "OpenTelemetry Profiling" +type: input +categories: + - custom + - observability + - opentelemetry +conditions: + kibana: + version: "^9.4.0" + elastic: + subscription: "basic" +policy_templates: + - name: profilingreceiver + type: profiles + title: OpenTelemetry Profiling receiver + description: Collect profiling data + input: otelcol + template_path: input.yml.hbs + vars: + - name: samples_per_second + type: integer + required: false + description: Defines the sampling frequency. +owner: + github: elastic/ingest-otel-data + type: elastic diff --git a/test/packages/good_v3/data_stream/foo/manifest.yml b/test/packages/good_v3/data_stream/foo/manifest.yml index 4982be628..f6317bdb5 100644 --- a/test/packages/good_v3/data_stream/foo/manifest.yml +++ b/test/packages/good_v3/data_stream/foo/manifest.yml @@ -63,6 +63,9 @@ elasticsearch: dimension_fields: limit: 32 number_of_shards: 1 + auto_expand_replicas: "0-1" + max_result_window: 150000 + refresh_interval: "10s" mappings: dynamic: strict dynamic_templates: