Otel profiles input package support

code/go/pkg/validator/validator_test.go

@@ -37,6 +37,7 @@ func TestValidateFile(t *testing.T) {
 		"good_input":                             {},
 		"good_input_otel":                        {},
 		"good_input_dynamic_signal_type":         {},
+		"good_input_profiles":                    {},
 		"good_input_template_paths":              {},
 		"good_integration_template_paths":        {},
 		"good_content":                           {},
@@ -390,6 +391,12 @@ func TestValidateFile(t *testing.T) {
 				"field policy_templates.0.input: Must not be present",
 			},
 		},
+		"bad_input_profiles_non_otel": {
+			"manifest.yml",
+			[]string{
+				"field policy_templates.0.input: policy_templates.0.input must be one of the following: \"otelcol\"",
+			},
+		},
 		"bad_input_dynamic_signal_types_non_otel": {
 			"manifest.yml",
 			[]string{

compliance/features/basic.feature

@@ -40,3 +40,9 @@ Feature: Basic package types support
    Given the "good_v3" package is installed
      And a policy is created with "good_v3" package, "1.1.0" version, "otel" template, "otelcol" input type and dataset "good_v3.otel_test"
     Then there is an index template "logs-good_v3.otel_test" with pattern "logs-good_v3.otel_test.otel-*"
+
+  @3.6.0
+  Scenario: OTel input package with profiles type can be installed
+   Given the "good_input_profiles" package is installed
+     And a policy is created with "good_input_profiles" package, "0.0.1" version, "profilingreceiver" template, "profilingreceiver" input, "otelcol" input type and dataset "spec.otel_input_test"
+     # We omit assertions here because 'profiles' OTel packages should not produce ES assets which is currently handled by an ES plugin. https://github.com/elastic/package-spec/pull/1092#discussion_r2857893043

spec/changelog.yml

@@ -58,6 +58,12 @@
     - description: Allow to force the format used in the policy API in system and policy tests.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/1103
+    - description: Add support for auto_expand_replicas, max_result_window, and refresh_interval index settings in data stream templates.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/1092
+    - description: Add support for `profiles` policy template type in input packages
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/1092
 - version: 3.5.7
   changes:
     - description: Allow _dev directory for content-only packages; use _dev/shared for development files (e.g. dashboard YML sources).

spec/input/manifest.spec.yml

@@ -55,6 +55,7 @@ spec:
               - logs
               - synthetics
               - traces
+              - profiles
             examples:
               - metrics
           title:
@@ -96,23 +97,36 @@ spec:
         allOf:
           # Either template_path or template_paths must be present (mutually exclusive)
           - oneOf:
-              - required:
-                  - template_path
-                not:
-                  required:
-                    - template_paths
-              - required:
+            - required:
+                - template_path
+              not:
+                required:
                   - template_paths
-                not:
-                  required:
-                    - template_path
+            - required:
+                - template_paths
+              not:
+                required:
+                  - template_path
           # Either type is required, or dynamic_signal_types must be true
           - oneOf:
+            - allOf:
               - required:
-                  - type
-              - properties:
-                  dynamic_signal_types:
-                    const: true
+                - type
+              # If type is profiles, can only use otelcol inputs
+              - if:
+                  properties:
+                    type:
+                      const: profiles
+                then:
+                  properties:
+                    input:
+                      enum:
+                        - otelcol
+            - required:
+                - dynamic_signal_types
+              properties:
+                dynamic_signal_types:
+                  const: true
         required:
           - name
           - title
@@ -155,7 +169,7 @@ versions:
       - op: remove
         path: "/properties/policy_templates/items/properties/template_paths" # removes template_paths field for policy templates
       - op: remove
-        path: "/properties/policy_templates/items/allOf" # removes allOf constraint for template_path/template_paths and dynamic_signal_types/type
+        path: "/properties/policy_templates/items/allOf" # removes allOf constraint for template_path/template_paths, dynamic_signal_types/type, and profiles otelcol input
       - op: add
         path: "/properties/policy_templates/items/required"
         value:
@@ -173,6 +187,8 @@ versions:
         path: "/properties/deprecated" # removes deprecated field for package
       - op: remove
         path: "/properties/policy_templates/items/properties/deprecated" # removes deprecated field for policy templates
+      - op: remove
+        path: "/properties/policy_templates/items/properties/type/enum/4" # removes 'profiles' as a valid policy template type, since it requires Kibana version >= 9.4.0 support
   # Reserve otelcol input name before 3.5.0.
   - before: 3.5.0
     patch:

spec/integration/data_stream/manifest.spec.yml

@@ -263,6 +263,12 @@ spec:
               $ref: "#/definitions/index_template_setting_index"
             number_of_shards:
               $ref: "#/definitions/index_template_setting_number_of_shards"
+            auto_expand_replicas:
+              $ref: "#/definitions/index_template_setting_auto_expand_replicas"
+            max_result_window:
+              $ref: "#/definitions/index_template_setting_max_result_window"
+            refresh_interval:
+              $ref: "#/definitions/index_template_setting_refresh_interval"
         mappings:
           description: Mappings section of index template
           type: object
@@ -512,6 +518,32 @@ spec:
       type: integer
       minimum: 1
       default: 1
+    index_template_setting_auto_expand_replicas:
+      description: |
+        Auto-expand the number of replicas based on the number of data nodes in the cluster.
+        Set to a dash delimited lower and upper bound (e.g. 0-5) or use `all` for the upper bound (e.g. 0-all). Defaults to false (i.e. disabled).
+      type: string
+      examples:
+        - "0-1"
+        - "0-all"
+        - "false"
+    index_template_setting_max_result_window:
+      description: |
+        The maximum value of `from + size` for searches to this index. Defaults to 10000.
+        Search requests take heap memory and time proportional to `from + size` and this limits that memory.
+      type: integer
+      minimum: 1
+      examples:
+        - 50000
+    index_template_setting_refresh_interval:
+      description: |
+        How often to perform a refresh operation, which makes recent changes to the index visible to search. The value defaults to `1s` in Elastic Stack and `5s` in Serverless. In Serverless, `5s` is also the minimum value that can be set.
+        Can be set to `-1` to disable refresh.
+      type: string
+      examples:
+        - "10s"
+        - "1m"
+        - "-1"
   properties:
     dataset:
       description: Name of data set.

test/packages/bad_input_profiles_non_otel/agent/input/input.yml.hbs

@@ -0,0 +1,10 @@
+receivers:
+  profiling:
+{{#if samples_per_second}}
+    samples_per_second: {{samples_per_second}}
+{{/if}}
+
+service:
+  pipelines:
+    profiles:
+      receivers: [ profiling ]

test/packages/bad_input_profiles_non_otel/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.0.1
+  changes:
+    - type: enhancement
+      description: Initial release for testing input policy templates with type `profiles` with a non otelcol input type.
+      link: https://github.com/elastic/package-spec/pull/1092

test/packages/bad_input_profiles_non_otel/docs/README.md

@@ -0,0 +1 @@
+This is a template for the package README.

test/packages/bad_input_profiles_non_otel/manifest.yml

@@ -0,0 +1,32 @@
+format_version: 3.6.0
+name: bad_input_profiles_non_otel
+title: "Bad OpenTelemetry Profiling"
+version: 0.0.1
+source:
+  license: "Elastic-2.0"
+description: "Bad OpenTelemetry Profiling"
+type: input
+categories:
+  - custom
+  - observability
+  - opentelemetry
+conditions:
+  kibana:
+    version: "^9.4.0"
+  elastic:
+    subscription: "basic"
+policy_templates:
+  - name: profilingreceiver
+    type: profiles
+    title: OpenTelemetry Profiling receiver
+    description: Collect profiling data
+    input: logs
+    template_path: input.yml.hbs
+    vars:
+      - name: samples_per_second
+        type: integer
+        required: false
+        description: Defines the sampling frequency.
+owner:
+  github: elastic/ingest-otel-data
+  type: elastic

test/packages/good_input_profiles/agent/input/input.yml.hbs

@@ -0,0 +1,10 @@
+receivers:
+  profiling:
+{{#if samples_per_second}}
+    samples_per_second: {{samples_per_second}}
+{{/if}}
+
+service:
+  pipelines:
+    profiles:
+      receivers: [ profiling ]

test/packages/good_input_profiles/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.0.1
+  changes:
+    - type: enhancement
+      description: Initial release for testing input policy templates with type `profiles`.
+      link: https://github.com/elastic/package-spec/pull/1092

test/packages/good_input_profiles/docs/README.md

@@ -0,0 +1 @@
+This is a template for the package README.

test/packages/good_input_profiles/manifest.yml

@@ -0,0 +1,32 @@
+format_version: 3.6.0
+name: good_input_profiles
+title: "OpenTelemetry Profiling"
+version: 0.0.1
+source:
+  license: "Elastic-2.0"
+description: "OpenTelemetry Profiling"
+type: input
+categories:
+  - custom
+  - observability
+  - opentelemetry
+conditions:
+  kibana:
+    version: "^9.4.0"
+  elastic:
+    subscription: "basic"
+policy_templates:
+  - name: profilingreceiver
+    type: profiles
+    title: OpenTelemetry Profiling receiver
+    description: Collect profiling data
+    input: otelcol
+    template_path: input.yml.hbs
+    vars:
+      - name: samples_per_second
+        type: integer
+        required: false
+        description: Defines the sampling frequency.
+owner:
+  github: elastic/ingest-otel-data
+  type: elastic

test/packages/good_v3/data_stream/foo/manifest.yml

@@ -63,6 +63,9 @@ elasticsearch:
           dimension_fields:
             limit: 32
       number_of_shards: 1
+      auto_expand_replicas: "0-1"
+      max_result_window: 150000
+      refresh_interval: "10s"
     mappings:
       dynamic: strict
       dynamic_templates: