Add more generic support for var_groups

code/go/pkg/validator/validator_test.go

@@ -33,6 +33,7 @@ func TestValidateFile(t *testing.T) {
 		"good_v2":                                {},
 		"good_v3":                                {},
 		"good_var_groups":                        {},
+		"good_var_groups_input":                  {},
 		"good_input":                             {},
 		"good_input_otel":                        {},
 		"good_input_dynamic_signal_type":         {},

spec/changelog.yml

@@ -24,6 +24,9 @@
     - description: Add var_groups schema to support conditional variable groups for Cloud Connector integration.
       type: enhancement
       link: https://github.com/elastic/package-spec/issues/1054
+    - description: Add var_groups support to policy template and input levels in integration packages, and to policy template and package root levels in input packages.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/PLACEHOLDER
     - description: Allow to set time series index mode in input packages.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/1066

spec/input/manifest.spec.yml

@@ -80,6 +80,8 @@ spec:
             $ref: "../integration/manifest.spec.yml#/definitions/screenshots"
           vars:
             $ref: "../integration/data_stream/manifest.spec.yml#/definitions/vars"
+          var_groups:
+            $ref: "../integration/manifest.spec.yml#/definitions/var_groups"
           input:
             type: string
             examples:
@@ -138,6 +140,8 @@ spec:
       $ref: "../integration/manifest.spec.yml#/definitions/screenshots"
     vars:
       $ref: "../integration/data_stream/manifest.spec.yml#/definitions/vars"
+    var_groups:
+      $ref: "../integration/manifest.spec.yml#/definitions/var_groups"
     owner:
       $ref: "../integration/manifest.spec.yml#/definitions/owner"
     agent:
@@ -166,6 +170,10 @@ spec:
 versions:
   - before: 3.6.0
     patch:
+      - op: remove
+        path: "/properties/var_groups" # removes var_groups from package root
+      - op: remove
+        path: "/properties/policy_templates/items/properties/var_groups" # removes var_groups from policy templates
       - op: remove
         path: "/properties/policy_templates/items/properties/template_paths" # removes template_paths field for policy templates
       - op: remove

spec/integration/manifest.spec.yml

@@ -830,6 +830,8 @@ spec:
                   $ref: "./data_stream/manifest.spec.yml#/definitions/required_vars"
                 vars:
                   $ref: "./data_stream/manifest.spec.yml#/definitions/vars"
+                var_groups:
+                  $ref: "#/definitions/var_groups"
                 dynamic_signal_types:
                   $ref: "#/definitions/dynamic_signal_types"
                 deprecated:
@@ -859,6 +861,8 @@ spec:
             $ref: "#/definitions/screenshots"
           vars:
             $ref: "./data_stream/manifest.spec.yml#/definitions/vars"
+          var_groups:
+            $ref: "#/definitions/var_groups"
           deprecated:
             $ref: "#/definitions/deprecated"
         required:
@@ -974,6 +978,10 @@ versions:
         path: "/definitions/var_groups"
       - op: remove
         path: "/properties/var_groups"
+      - op: remove
+        path: "/properties/policy_templates/items/properties/var_groups"
+      - op: remove
+        path: "/properties/policy_templates/items/properties/inputs/items/properties/var_groups"
       - op: remove
         path: "/properties/policy_templates/items/properties/inputs/items/properties/hide_in_var_group_options"
       # Input migration.

test/packages/good_var_groups/manifest.yml

@@ -87,6 +87,35 @@ policy_templates:
     description: Collect sample logs and metrics
     data_streams:
       - findings
+    vars:
+      - name: username
+        type: text
+        title: Username
+        show_user: true
+      - name: password
+        type: password
+        title: Password
+        show_user: true
+        secret: true
+      - name: token
+        type: password
+        title: API Token
+        show_user: true
+        secret: true
+    var_groups:
+      - name: auth_method
+        title: Authentication Method
+        selector_title: Authentication method
+        options:
+          - name: basic
+            title: Basic Auth
+            vars:
+              - username
+              - password
+          - name: token
+            title: Token Auth
+            vars:
+              - token
     inputs:
       - type: httpjson
         title: Collect via API
@@ -98,6 +127,29 @@ policy_templates:
             title: API URL
             show_user: true
             required: true
+          - name: bearer_token
+            type: password
+            title: Bearer Token
+            show_user: true
+            secret: true
+          - name: api_key
+            type: password
+            title: API Key
+            show_user: true
+            secret: true
+        var_groups:
+          - name: auth_method
+            title: Authentication
+            selector_title: Authentication method
+            options:
+              - name: bearer
+                title: Bearer Token
+                vars:
+                  - bearer_token
+              - name: api_key
+                title: API Key
+                vars:
+                  - api_key
       - type: aws-s3
         title: Collect from S3
         description: Collecting logs from AWS S3 bucket

test/packages/good_var_groups_input/LICENSE.txt

@@ -0,0 +1 @@
+Apache License 2.0

test/packages/good_var_groups_input/agent/input/input.yml.hbs

@@ -0,0 +1,9 @@
+data_stream:
+  dataset: {{data_stream.dataset}}
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~

test/packages/good_var_groups_input/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1

test/packages/good_var_groups_input/docs/README.md

@@ -0,0 +1,3 @@
+# Good Var Groups Input Package
+
+This is a test package demonstrating valid `var_groups` usage in an input package at the package root and policy template levels.

test/packages/good_var_groups_input/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/good_var_groups_input/manifest.yml

@@ -0,0 +1,86 @@
+format_version: 3.6.0
+name: good_var_groups_input
+title: Good Var Groups Input Package
+description: This package demonstrates valid var_groups usage in an input package.
+version: 0.0.1
+type: input
+source:
+  license: "Apache-2.0"
+conditions:
+  kibana:
+    version: "^8.10.0"
+  elastic:
+    subscription: "basic"
+icons:
+  - src: /img/sample-logo.svg
+    title: Sample logo
+    size: 32x32
+    type: image/svg+xml
+vars:
+  - name: access_key
+    type: text
+    title: Access Key
+    show_user: true
+    secret: false
+  - name: secret_key
+    type: password
+    title: Secret Key
+    show_user: true
+    secret: true
+  - name: iam_role
+    type: text
+    title: IAM Role ARN
+    show_user: true
+var_groups:
+  - name: credential_type
+    title: Credential Type
+    selector_title: Preferred method
+    options:
+      - name: access_key
+        title: Access Key
+        vars:
+          - access_key
+          - secret_key
+      - name: iam_role
+        title: IAM Role
+        vars:
+          - iam_role
+policy_templates:
+  - name: logs
+    type: logs
+    title: Logs
+    description: Collect logs.
+    input: logfile
+    template_path: input.yml.hbs
+    vars:
+      - name: username
+        type: text
+        title: Username
+        show_user: true
+      - name: password
+        type: password
+        title: Password
+        show_user: true
+        secret: true
+      - name: token
+        type: password
+        title: API Token
+        show_user: true
+        secret: true
+    var_groups:
+      - name: auth_method
+        title: Authentication Method
+        selector_title: Authentication method
+        options:
+          - name: basic
+            title: Basic Auth
+            vars:
+              - username
+              - password
+          - name: token
+            title: Token Auth
+            vars:
+              - token
+owner:
+  github: elastic/ecosystem
+  type: elastic