From b7352ebb1d08b83da1fe87d65cae40f2bfb3c549 Mon Sep 17 00:00:00 2001 From: Tere Date: Tue, 10 Mar 2026 14:26:13 +0100 Subject: [PATCH 1/4] add test packages for variables merge --- .../agent/input/input.yml.hbs | 4 ++ .../var_merging_input_pkg/changelog.yml | 5 +++ .../var_merging_input_pkg/docs/README.md | 3 ++ .../fields/base-fields.yml | 12 +++++ .../var_merging_input_pkg/manifest.yml | 45 +++++++++++++++++++ .../_dev/test/config.yml | 3 ++ .../with_merging_ds_merges/changelog.yml | 5 +++ .../agent/stream/stream.yml.hbs | 4 ++ .../var_merging_logs/fields/base-fields.yml | 12 +++++ .../data_stream/var_merging_logs/manifest.yml | 13 ++++++ .../with_merging_ds_merges/docs/README.md | 13 ++++++ .../with_merging_ds_merges/manifest.yml | 33 ++++++++++++++ .../_dev/test/config.yml | 3 ++ .../changelog.yml | 5 +++ .../agent/stream/stream.yml.hbs | 4 ++ .../var_merging_logs/fields/base-fields.yml | 12 +++++ .../data_stream/var_merging_logs/manifest.yml | 11 +++++ .../docs/README.md | 8 ++++ .../with_merging_duplicate_error/manifest.yml | 33 ++++++++++++++ .../with_merging_full/_dev/test/config.yml | 3 ++ .../with_merging_full/changelog.yml | 5 +++ .../agent/stream/stream.yml.hbs | 4 ++ .../var_merging_logs/fields/base-fields.yml | 12 +++++ .../data_stream/var_merging_logs/manifest.yml | 13 ++++++ .../with_merging_full/docs/README.md | 20 +++++++++ .../with_merging_full/manifest.yml | 40 +++++++++++++++++ .../_dev/test/config.yml | 3 ++ .../with_merging_no_override/changelog.yml | 5 +++ .../agent/stream/stream.yml.hbs | 4 ++ .../var_merging_logs/fields/base-fields.yml | 12 +++++ .../data_stream/var_merging_logs/manifest.yml | 6 +++ .../with_merging_no_override/docs/README.md | 5 +++ .../with_merging_no_override/manifest.yml | 32 +++++++++++++ .../_dev/test/config.yml | 3 ++ .../changelog.yml | 5 +++ .../agent/stream/stream.yml.hbs | 4 ++ .../var_merging_logs/fields/base-fields.yml | 12 +++++ .../data_stream/var_merging_logs/manifest.yml | 6 +++ .../docs/README.md | 11 +++++ .../manifest.yml | 38 ++++++++++++++++ 40 files changed, 471 insertions(+) create mode 100644 test/packages/required_inputs/var_merging_input_pkg/agent/input/input.yml.hbs create mode 100644 test/packages/required_inputs/var_merging_input_pkg/changelog.yml create mode 100644 test/packages/required_inputs/var_merging_input_pkg/docs/README.md create mode 100644 test/packages/required_inputs/var_merging_input_pkg/fields/base-fields.yml create mode 100644 test/packages/required_inputs/var_merging_input_pkg/manifest.yml create mode 100644 test/packages/required_inputs/with_merging_ds_merges/_dev/test/config.yml create mode 100644 test/packages/required_inputs/with_merging_ds_merges/changelog.yml create mode 100644 test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/agent/stream/stream.yml.hbs create mode 100644 test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/fields/base-fields.yml create mode 100644 test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/manifest.yml create mode 100644 test/packages/required_inputs/with_merging_ds_merges/docs/README.md create mode 100644 test/packages/required_inputs/with_merging_ds_merges/manifest.yml create mode 100644 test/packages/required_inputs/with_merging_duplicate_error/_dev/test/config.yml create mode 100644 test/packages/required_inputs/with_merging_duplicate_error/changelog.yml create mode 100644 test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/agent/stream/stream.yml.hbs create mode 100644 test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/fields/base-fields.yml create mode 100644 test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/manifest.yml create mode 100644 test/packages/required_inputs/with_merging_duplicate_error/docs/README.md create mode 100644 test/packages/required_inputs/with_merging_duplicate_error/manifest.yml create mode 100644 test/packages/required_inputs/with_merging_full/_dev/test/config.yml create mode 100644 test/packages/required_inputs/with_merging_full/changelog.yml create mode 100644 test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/agent/stream/stream.yml.hbs create mode 100644 test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/fields/base-fields.yml create mode 100644 test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/manifest.yml create mode 100644 test/packages/required_inputs/with_merging_full/docs/README.md create mode 100644 test/packages/required_inputs/with_merging_full/manifest.yml create mode 100644 test/packages/required_inputs/with_merging_no_override/_dev/test/config.yml create mode 100644 test/packages/required_inputs/with_merging_no_override/changelog.yml create mode 100644 test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/agent/stream/stream.yml.hbs create mode 100644 test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/fields/base-fields.yml create mode 100644 test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/manifest.yml create mode 100644 test/packages/required_inputs/with_merging_no_override/docs/README.md create mode 100644 test/packages/required_inputs/with_merging_no_override/manifest.yml create mode 100644 test/packages/required_inputs/with_merging_promotes_to_input/_dev/test/config.yml create mode 100644 test/packages/required_inputs/with_merging_promotes_to_input/changelog.yml create mode 100644 test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/agent/stream/stream.yml.hbs create mode 100644 test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/fields/base-fields.yml create mode 100644 test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/manifest.yml create mode 100644 test/packages/required_inputs/with_merging_promotes_to_input/docs/README.md create mode 100644 test/packages/required_inputs/with_merging_promotes_to_input/manifest.yml diff --git a/test/packages/required_inputs/var_merging_input_pkg/agent/input/input.yml.hbs b/test/packages/required_inputs/var_merging_input_pkg/agent/input/input.yml.hbs new file mode 100644 index 0000000000..9e9c27a8c0 --- /dev/null +++ b/test/packages/required_inputs/var_merging_input_pkg/agent/input/input.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} \ No newline at end of file diff --git a/test/packages/required_inputs/var_merging_input_pkg/changelog.yml b/test/packages/required_inputs/var_merging_input_pkg/changelog.yml new file mode 100644 index 0000000000..fb3f5f7235 --- /dev/null +++ b/test/packages/required_inputs/var_merging_input_pkg/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.1.0 + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/elastic-package/issues/1 \ No newline at end of file diff --git a/test/packages/required_inputs/var_merging_input_pkg/docs/README.md b/test/packages/required_inputs/var_merging_input_pkg/docs/README.md new file mode 100644 index 0000000000..894e4fe149 --- /dev/null +++ b/test/packages/required_inputs/var_merging_input_pkg/docs/README.md @@ -0,0 +1,3 @@ +# Var Merging Input Package + +Input package used as a test fixture for variable merging tests. \ No newline at end of file diff --git a/test/packages/required_inputs/var_merging_input_pkg/fields/base-fields.yml b/test/packages/required_inputs/var_merging_input_pkg/fields/base-fields.yml new file mode 100644 index 0000000000..d3b0f5a163 --- /dev/null +++ b/test/packages/required_inputs/var_merging_input_pkg/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. \ No newline at end of file diff --git a/test/packages/required_inputs/var_merging_input_pkg/manifest.yml b/test/packages/required_inputs/var_merging_input_pkg/manifest.yml new file mode 100644 index 0000000000..0e315aa0de --- /dev/null +++ b/test/packages/required_inputs/var_merging_input_pkg/manifest.yml @@ -0,0 +1,45 @@ +format_version: 3.6.0 +name: var_merging_input_pkg +title: Var Merging Input Package +description: Input package used as a test fixture for variable merging. +version: 0.1.0 +type: input +categories: + - custom +conditions: + kibana: + version: "^8.0.0" + elastic: + subscription: basic +policy_templates: + - name: var_merging + type: logs + title: Var Merging + description: Collect logs with multiple variables. + input: logfile + template_path: input.yml.hbs + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/*.log + - name: encoding + type: text + title: Encoding + multi: false + required: false + show_user: false + - name: timeout + type: text + title: Timeout + multi: false + required: false + show_user: false + default: 30s +owner: + github: elastic/integrations + type: elastic \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_ds_merges/_dev/test/config.yml b/test/packages/required_inputs/with_merging_ds_merges/_dev/test/config.yml new file mode 100644 index 0000000000..bbb3460521 --- /dev/null +++ b/test/packages/required_inputs/with_merging_ds_merges/_dev/test/config.yml @@ -0,0 +1,3 @@ +requires: + - package: var_merging_input_pkg + source: "../../var_merging_input_pkg" \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_ds_merges/changelog.yml b/test/packages/required_inputs/with_merging_ds_merges/changelog.yml new file mode 100644 index 0000000000..fb3f5f7235 --- /dev/null +++ b/test/packages/required_inputs/with_merging_ds_merges/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.1.0 + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/elastic-package/issues/1 \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/agent/stream/stream.yml.hbs b/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..9e9c27a8c0 --- /dev/null +++ b/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/fields/base-fields.yml b/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/fields/base-fields.yml new file mode 100644 index 0000000000..d3b0f5a163 --- /dev/null +++ b/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/manifest.yml b/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/manifest.yml new file mode 100644 index 0000000000..589b1e1604 --- /dev/null +++ b/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/manifest.yml @@ -0,0 +1,13 @@ +title: Var Merging Logs +type: logs +streams: + - package: var_merging_input_pkg + title: Var Merging Logs + description: Collect logs using the var merging input package. + vars: + - name: encoding + title: Log Encoding Override + - name: custom_tag + type: text + title: Custom Tag + show_user: true \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_ds_merges/docs/README.md b/test/packages/required_inputs/with_merging_ds_merges/docs/README.md new file mode 100644 index 0000000000..db7779e04e --- /dev/null +++ b/test/packages/required_inputs/with_merging_ds_merges/docs/README.md @@ -0,0 +1,13 @@ +# Variable Merging - Data Stream Merges + +Test fixture: composable package with no policy template variable overrides. +The data stream manifest overrides the "encoding" variable from the input +package (providing a different title) and adds a new "custom_tag" variable. + +Expected result after merging: +- Input variables: (none) +- Data stream variables: + - paths (unchanged from input package) + - encoding (merged: base from input pkg, title overridden) + - timeout (unchanged from input package) + - custom_tag (new, from data stream manifest) \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_ds_merges/manifest.yml b/test/packages/required_inputs/with_merging_ds_merges/manifest.yml new file mode 100644 index 0000000000..9ad6c0e1a8 --- /dev/null +++ b/test/packages/required_inputs/with_merging_ds_merges/manifest.yml @@ -0,0 +1,33 @@ +format_version: 3.6.0 +name: with_merging_ds_merges +title: Variable Merging - Data Stream Merges +description: >- + Composable package with no policy template variable overrides. The data stream + manifest overrides the "encoding" variable (changing its title) and introduces + a new "custom_tag" variable. All variables remain in the data stream list. +version: 0.1.0 +type: integration +categories: + - custom +conditions: + kibana: + version: "^8.0.0" + elastic: + subscription: basic +requires: + input: + - package: var_merging_input_pkg + version: "0.1.0" +policy_templates: + - name: var_merging_logs + title: Var Merging Logs + description: Collect logs via var merging input package + data_streams: + - var_merging_logs + inputs: + - package: var_merging_input_pkg + title: Collect logs via var merging input package + description: Use the var merging input package to collect logs +owner: + github: elastic/integrations + type: elastic \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_duplicate_error/_dev/test/config.yml b/test/packages/required_inputs/with_merging_duplicate_error/_dev/test/config.yml new file mode 100644 index 0000000000..bbb3460521 --- /dev/null +++ b/test/packages/required_inputs/with_merging_duplicate_error/_dev/test/config.yml @@ -0,0 +1,3 @@ +requires: + - package: var_merging_input_pkg + source: "../../var_merging_input_pkg" \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_duplicate_error/changelog.yml b/test/packages/required_inputs/with_merging_duplicate_error/changelog.yml new file mode 100644 index 0000000000..fb3f5f7235 --- /dev/null +++ b/test/packages/required_inputs/with_merging_duplicate_error/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.1.0 + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/elastic-package/issues/1 \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/agent/stream/stream.yml.hbs b/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..9e9c27a8c0 --- /dev/null +++ b/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/fields/base-fields.yml b/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/fields/base-fields.yml new file mode 100644 index 0000000000..d3b0f5a163 --- /dev/null +++ b/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/manifest.yml b/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/manifest.yml new file mode 100644 index 0000000000..f7b06783dd --- /dev/null +++ b/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/manifest.yml @@ -0,0 +1,11 @@ +title: Var Merging Logs +type: logs +streams: + - package: var_merging_input_pkg + title: Var Merging Logs + description: Collect logs using the var merging input package. + vars: + - name: paths + title: First paths definition + - name: paths + title: Duplicate paths definition \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_duplicate_error/docs/README.md b/test/packages/required_inputs/with_merging_duplicate_error/docs/README.md new file mode 100644 index 0000000000..83f27f9389 --- /dev/null +++ b/test/packages/required_inputs/with_merging_duplicate_error/docs/README.md @@ -0,0 +1,8 @@ +# Variable Merging - Duplicate Error + +Test fixture: composable package whose data stream manifest defines the "paths" +variable twice. The merging algorithm must detect this duplicate and return an +error (Step 5: fail if there are multiple variables with the same name). + +Expected result: error indicating a duplicate variable name "paths" in the data +stream variable list. \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_duplicate_error/manifest.yml b/test/packages/required_inputs/with_merging_duplicate_error/manifest.yml new file mode 100644 index 0000000000..64221efbaa --- /dev/null +++ b/test/packages/required_inputs/with_merging_duplicate_error/manifest.yml @@ -0,0 +1,33 @@ +format_version: 3.6.0 +name: with_merging_duplicate_error +title: Variable Merging - Duplicate Error +description: >- + Composable package whose data stream manifest defines the "paths" variable + twice. This should cause the variable merging step to fail with a duplicate + variable name error. +version: 0.1.0 +type: integration +categories: + - custom +conditions: + kibana: + version: "^8.0.0" + elastic: + subscription: basic +requires: + input: + - package: var_merging_input_pkg + version: "0.1.0" +policy_templates: + - name: var_merging_logs + title: Var Merging Logs + description: Collect logs via var merging input package + data_streams: + - var_merging_logs + inputs: + - package: var_merging_input_pkg + title: Collect logs via var merging input package + description: Use the var merging input package to collect logs +owner: + github: elastic/integrations + type: elastic \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_full/_dev/test/config.yml b/test/packages/required_inputs/with_merging_full/_dev/test/config.yml new file mode 100644 index 0000000000..bbb3460521 --- /dev/null +++ b/test/packages/required_inputs/with_merging_full/_dev/test/config.yml @@ -0,0 +1,3 @@ +requires: + - package: var_merging_input_pkg + source: "../../var_merging_input_pkg" \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_full/changelog.yml b/test/packages/required_inputs/with_merging_full/changelog.yml new file mode 100644 index 0000000000..fb3f5f7235 --- /dev/null +++ b/test/packages/required_inputs/with_merging_full/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.1.0 + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/elastic-package/issues/1 \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/agent/stream/stream.yml.hbs b/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..9e9c27a8c0 --- /dev/null +++ b/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/fields/base-fields.yml b/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/fields/base-fields.yml new file mode 100644 index 0000000000..d3b0f5a163 --- /dev/null +++ b/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/manifest.yml b/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/manifest.yml new file mode 100644 index 0000000000..d3a6f017b0 --- /dev/null +++ b/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/manifest.yml @@ -0,0 +1,13 @@ +title: Var Merging Logs +type: logs +streams: + - package: var_merging_input_pkg + title: Var Merging Logs + description: Collect logs using the var merging input package. + vars: + - name: timeout + description: Timeout for log collection. + - name: custom_tag + type: text + title: Custom Tag + show_user: true \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_full/docs/README.md b/test/packages/required_inputs/with_merging_full/docs/README.md new file mode 100644 index 0000000000..2900be833f --- /dev/null +++ b/test/packages/required_inputs/with_merging_full/docs/README.md @@ -0,0 +1,20 @@ +# Variable Merging - Full Mix + +Test fixture: composable package that exercises all five variable merging steps +from SPEC.md simultaneously. + +Policy template input vars (Step 2 → Step 3 promotion): +- "paths" override with new default → promoted to input variable +- "encoding" override with show_user:true → promoted to input variable + +Data stream manifest vars (Step 4 merge): +- "timeout" override with new description → merged with remaining DS variable +- "custom_tag" new variable → added to DS variables + +Expected result after merging: +- Input variables: + - paths (merged: base from input pkg, default overridden to /var/log/custom/*.log) + - encoding (merged: base from input pkg, show_user overridden to true) +- Data stream variables: + - timeout (merged: base from input pkg, description overridden) + - custom_tag (new, from data stream manifest) \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_full/manifest.yml b/test/packages/required_inputs/with_merging_full/manifest.yml new file mode 100644 index 0000000000..6b617324f5 --- /dev/null +++ b/test/packages/required_inputs/with_merging_full/manifest.yml @@ -0,0 +1,40 @@ +format_version: 3.6.0 +name: with_merging_full +title: Variable Merging - Full Mix +description: >- + Composable package exercising all variable merging steps. The policy template + overrides "paths" and "encoding" (both promoted to input variables). The data + stream manifest overrides "timeout" (merged with the remaining data stream + variable) and adds "custom_tag" (new data stream variable). +version: 0.1.0 +type: integration +categories: + - custom +conditions: + kibana: + version: "^8.0.0" + elastic: + subscription: basic +requires: + input: + - package: var_merging_input_pkg + version: "0.1.0" +policy_templates: + - name: var_merging_logs + title: Var Merging Logs + description: Collect logs via var merging input package + data_streams: + - var_merging_logs + inputs: + - package: var_merging_input_pkg + title: Collect logs via var merging input package + description: Use the var merging input package to collect logs + vars: + - name: paths + default: + - /var/log/custom/*.log + - name: encoding + show_user: true +owner: + github: elastic/integrations + type: elastic \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_no_override/_dev/test/config.yml b/test/packages/required_inputs/with_merging_no_override/_dev/test/config.yml new file mode 100644 index 0000000000..bbb3460521 --- /dev/null +++ b/test/packages/required_inputs/with_merging_no_override/_dev/test/config.yml @@ -0,0 +1,3 @@ +requires: + - package: var_merging_input_pkg + source: "../../var_merging_input_pkg" \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_no_override/changelog.yml b/test/packages/required_inputs/with_merging_no_override/changelog.yml new file mode 100644 index 0000000000..fb3f5f7235 --- /dev/null +++ b/test/packages/required_inputs/with_merging_no_override/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.1.0 + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/elastic-package/issues/1 \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/agent/stream/stream.yml.hbs b/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..9e9c27a8c0 --- /dev/null +++ b/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/fields/base-fields.yml b/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/fields/base-fields.yml new file mode 100644 index 0000000000..d3b0f5a163 --- /dev/null +++ b/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/manifest.yml b/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/manifest.yml new file mode 100644 index 0000000000..2026cd129c --- /dev/null +++ b/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/manifest.yml @@ -0,0 +1,6 @@ +title: Var Merging Logs +type: logs +streams: + - package: var_merging_input_pkg + title: Var Merging Logs + description: Collect logs using the var merging input package. \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_no_override/docs/README.md b/test/packages/required_inputs/with_merging_no_override/docs/README.md new file mode 100644 index 0000000000..cd0cb48e30 --- /dev/null +++ b/test/packages/required_inputs/with_merging_no_override/docs/README.md @@ -0,0 +1,5 @@ +# Variable Merging - No Override + +Test fixture: composable package with no variable overrides. All variables +defined in the input package policy template become data stream variables +unchanged. \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_no_override/manifest.yml b/test/packages/required_inputs/with_merging_no_override/manifest.yml new file mode 100644 index 0000000000..ebc0de82ac --- /dev/null +++ b/test/packages/required_inputs/with_merging_no_override/manifest.yml @@ -0,0 +1,32 @@ +format_version: 3.6.0 +name: with_merging_no_override +title: Variable Merging - No Override +description: >- + Composable package with no variable overrides at the policy template or data + stream level. All input package vars remain as data stream variables. +version: 0.1.0 +type: integration +categories: + - custom +conditions: + kibana: + version: "^8.0.0" + elastic: + subscription: basic +requires: + input: + - package: var_merging_input_pkg + version: "0.1.0" +policy_templates: + - name: var_merging_logs + title: Var Merging Logs + description: Collect logs via var merging input package + data_streams: + - var_merging_logs + inputs: + - package: var_merging_input_pkg + title: Collect logs via var merging input package + description: Use the var merging input package to collect logs +owner: + github: elastic/integrations + type: elastic \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/_dev/test/config.yml b/test/packages/required_inputs/with_merging_promotes_to_input/_dev/test/config.yml new file mode 100644 index 0000000000..bbb3460521 --- /dev/null +++ b/test/packages/required_inputs/with_merging_promotes_to_input/_dev/test/config.yml @@ -0,0 +1,3 @@ +requires: + - package: var_merging_input_pkg + source: "../../var_merging_input_pkg" \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/changelog.yml b/test/packages/required_inputs/with_merging_promotes_to_input/changelog.yml new file mode 100644 index 0000000000..fb3f5f7235 --- /dev/null +++ b/test/packages/required_inputs/with_merging_promotes_to_input/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.1.0 + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/elastic-package/issues/1 \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/agent/stream/stream.yml.hbs b/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..9e9c27a8c0 --- /dev/null +++ b/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/fields/base-fields.yml b/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/fields/base-fields.yml new file mode 100644 index 0000000000..d3b0f5a163 --- /dev/null +++ b/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/manifest.yml b/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/manifest.yml new file mode 100644 index 0000000000..2026cd129c --- /dev/null +++ b/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/manifest.yml @@ -0,0 +1,6 @@ +title: Var Merging Logs +type: logs +streams: + - package: var_merging_input_pkg + title: Var Merging Logs + description: Collect logs using the var merging input package. \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/docs/README.md b/test/packages/required_inputs/with_merging_promotes_to_input/docs/README.md new file mode 100644 index 0000000000..cead82918f --- /dev/null +++ b/test/packages/required_inputs/with_merging_promotes_to_input/docs/README.md @@ -0,0 +1,11 @@ +# Variable Merging - Promotes to Input Var + +Test fixture: composable package whose policy template declares a "paths" +variable override. Because "paths" is also defined in the input package policy +template, it is promoted from a data stream variable to an input variable and +merged (input package definition is the base; the override here changes the +default path). + +Expected result after merging: +- Input variables: paths (merged, default overridden to /var/log/custom/*.log) +- Data stream variables: encoding, timeout \ No newline at end of file diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/manifest.yml b/test/packages/required_inputs/with_merging_promotes_to_input/manifest.yml new file mode 100644 index 0000000000..c02a62f75d --- /dev/null +++ b/test/packages/required_inputs/with_merging_promotes_to_input/manifest.yml @@ -0,0 +1,38 @@ +format_version: 3.6.0 +name: with_merging_promotes_to_input +title: Variable Merging - Promotes to Input Var +description: >- + Composable package whose policy template overrides the "paths" variable from + the input package. This causes "paths" to be promoted from a data stream + variable to an input variable and merged. "encoding" and "timeout" remain as + data stream variables. +version: 0.1.0 +type: integration +categories: + - custom +conditions: + kibana: + version: "^8.0.0" + elastic: + subscription: basic +requires: + input: + - package: var_merging_input_pkg + version: "0.1.0" +policy_templates: + - name: var_merging_logs + title: Var Merging Logs + description: Collect logs via var merging input package + data_streams: + - var_merging_logs + inputs: + - package: var_merging_input_pkg + title: Collect logs via var merging input package + description: Use the var merging input package to collect logs + vars: + - name: paths + default: + - /var/log/custom/*.log +owner: + github: elastic/integrations + type: elastic \ No newline at end of file From 5da8b5a38035343aa853bd3da4d9e5ca9d9a8dfe Mon Sep 17 00:00:00 2001 From: Tere Date: Mon, 23 Mar 2026 14:51:01 +0100 Subject: [PATCH 2/4] feat: merge variables from input packages into composable package manifests Implements variable merging as part of the required_inputs build step (issue #3279). Input package vars are the base; composable package overrides win when explicitly specified. Promoted input-level vars and data-stream-level vars are correctly placed in the merged manifest. Co-Authored-By: Claude Sonnet 4.6 --- internal/requiredinputs/requiredinputs.go | 4 + internal/requiredinputs/variables.go | 500 ++++++++++++++++++++++ internal/requiredinputs/variables_test.go | 419 ++++++++++++++++++ internal/requiredinputs/yamlutil.go | 16 + 4 files changed, 939 insertions(+) create mode 100644 internal/requiredinputs/variables.go create mode 100644 internal/requiredinputs/variables_test.go diff --git a/internal/requiredinputs/requiredinputs.go b/internal/requiredinputs/requiredinputs.go index 9271e8c596..1336172195 100644 --- a/internal/requiredinputs/requiredinputs.go +++ b/internal/requiredinputs/requiredinputs.go @@ -91,6 +91,10 @@ func (r *RequiredInputsResolver) BundleInputPackageTemplates(buildPackageRoot st return fmt.Errorf("failed to bundle data stream input package templates: %w", err) } + if err := r.mergeVariables(manifest, inputPkgPaths, buildRoot); err != nil { + return fmt.Errorf("merging variables from input packages: %w", err) + } + return nil } diff --git a/internal/requiredinputs/variables.go b/internal/requiredinputs/variables.go new file mode 100644 index 0000000000..c7561c068c --- /dev/null +++ b/internal/requiredinputs/variables.go @@ -0,0 +1,500 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package requiredinputs + +import ( + "fmt" + "io/fs" + "maps" + "os" + "path" + + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-package/internal/packages" +) + +// pkgDsKey uniquely identifies the (input-package, data-stream) pair used to +// index promoted variable overrides. +type pkgDsKey struct { + pkg string + dsName string +} + +// mergeVariables merges variable definitions from input packages into the +// composable package's manifests (package-level and data-stream-level). +// +// Merging rule: input package vars are the base; composable package override +// fields win when explicitly specified. +// +// Input-level vars: vars declared in policy_templates[].inputs[].vars are +// "promoted" — they become input-level variables in the merged manifest. +// +// Data-stream-level vars: all remaining (non-promoted) base vars are placed at +// the data-stream level, merged with any stream-level overrides the composable +// package declares. +func (r *RequiredInputsResolver) mergeVariables( + manifest *packages.PackageManifest, + inputPkgPaths map[string]string, + buildRoot *os.Root, +) error { + // Step A — Re-read manifest.yml from disk as a YAML node so edits from the + // earlier template-bundling step are included. + manifestBytes, err := buildRoot.ReadFile("manifest.yml") + if err != nil { + return fmt.Errorf("reading manifest: %w", err) + } + var doc yaml.Node + if err := yaml.Unmarshal(manifestBytes, &doc); err != nil { + return fmt.Errorf("parsing manifest YAML: %w", err) + } + + // Step B — Build a promotedIndex: (pkg, dsName) → map[varName]overrideNode. + // The override nodes come from policy_templates[ptIdx].inputs[inputIdx].vars + // in the composable package manifest. + promotedIndex := make(map[pkgDsKey]map[string]*yaml.Node) + for ptIdx, pt := range manifest.PolicyTemplates { + for inputIdx, input := range pt.Inputs { + if input.Package == "" || len(input.Vars) == 0 { + continue + } + + inputNode, err := getInputMappingNode(&doc, ptIdx, inputIdx) + if err != nil { + return fmt.Errorf("getting input node at pt[%d].inputs[%d]: %w", ptIdx, inputIdx, err) + } + + overrideNodes, err := readVarNodes(inputNode) + if err != nil { + return fmt.Errorf("reading override var nodes at pt[%d].inputs[%d]: %w", ptIdx, inputIdx, err) + } + + overrideByName := make(map[string]*yaml.Node, len(overrideNodes)) + for _, n := range overrideNodes { + overrideByName[varNodeName(n)] = n + } + + dsNames := pt.DataStreams + if len(dsNames) == 0 { + dsNames = []string{""} + } + for _, dsName := range dsNames { + promotedIndex[pkgDsKey{pkg: input.Package, dsName: dsName}] = overrideByName + } + } + } + + // Step C — Merge and write input-level vars in manifest.yml. + for ptIdx, pt := range manifest.PolicyTemplates { + for inputIdx, input := range pt.Inputs { + if input.Package == "" { + continue + } + pkgPath, ok := inputPkgPaths[input.Package] + if !ok { + continue + } + + baseVarOrder, baseVarByName, err := loadInputPkgVarNodes(pkgPath) + if err != nil { + return fmt.Errorf("loading input pkg var nodes for %q: %w", input.Package, err) + } + if len(baseVarOrder) == 0 { + continue + } + + // Union of promoted overrides across all data streams for this input. + promotedOverrides := make(map[string]*yaml.Node) + dsNames := pt.DataStreams + if len(dsNames) == 0 { + dsNames = []string{""} + } + for _, dsName := range dsNames { + maps.Copy(promotedOverrides, promotedIndex[pkgDsKey{pkg: input.Package, dsName: dsName}]) + } + + inputNode, err := getInputMappingNode(&doc, ptIdx, inputIdx) + if err != nil { + return fmt.Errorf("getting input node at pt[%d].inputs[%d]: %w", ptIdx, inputIdx, err) + } + + mergedSeq, err := mergeInputLevelVarNodes(baseVarOrder, baseVarByName, promotedOverrides) + if err != nil { + return fmt.Errorf("merging input-level vars for pt[%d].inputs[%d]: %w", ptIdx, inputIdx, err) + } + + if len(mergedSeq.Content) > 0 { + upsertKey(inputNode, "vars", mergedSeq) + } else { + removeKey(inputNode, "vars") + } + } + } + + // Step D — Write the updated manifest.yml back to disk. + updated, err := formatYAMLNode(&doc) + if err != nil { + return fmt.Errorf("formatting updated manifest: %w", err) + } + if err := buildRoot.WriteFile("manifest.yml", updated, 0664); err != nil { + return fmt.Errorf("writing updated manifest: %w", err) + } + + // Step E — Process each data_stream/*/manifest.yml. + dsManifestPaths, err := fs.Glob(buildRoot.FS(), "data_stream/*/manifest.yml") + if err != nil { + return fmt.Errorf("globbing data stream manifests: %w", err) + } + + for _, manifestPath := range dsManifestPaths { + // data_stream/var_merging_logs/manifest.yml → var_merging_logs + dsName := path.Base(path.Dir(manifestPath)) + + dsManifestBytes, err := buildRoot.ReadFile(manifestPath) + if err != nil { + return fmt.Errorf("reading data stream manifest %q: %w", manifestPath, err) + } + + var dsDoc yaml.Node + if err := yaml.Unmarshal(dsManifestBytes, &dsDoc); err != nil { + return fmt.Errorf("parsing data stream manifest YAML %q: %w", manifestPath, err) + } + + dsManifest, err := packages.ReadDataStreamManifestBytes(dsManifestBytes) + if err != nil { + return fmt.Errorf("parsing data stream manifest %q: %w", manifestPath, err) + } + + for streamIdx, stream := range dsManifest.Streams { + if stream.Package == "" { + continue + } + pkgPath, ok := inputPkgPaths[stream.Package] + if !ok { + continue + } + + baseVarOrder, baseVarByName, err := loadInputPkgVarNodes(pkgPath) + if err != nil { + return fmt.Errorf("loading input pkg var nodes for %q: %w", stream.Package, err) + } + if len(baseVarOrder) == 0 { + continue + } + + // Promoted names for this (pkg, dsName) combination. + promotedNames := make(map[string]bool) + for _, key := range []pkgDsKey{{stream.Package, dsName}, {stream.Package, ""}} { + for varName := range promotedIndex[key] { + promotedNames[varName] = true + } + } + + streamNode, err := getStreamMappingNode(&dsDoc, streamIdx) + if err != nil { + return fmt.Errorf("getting stream node at index %d in %q: %w", streamIdx, manifestPath, err) + } + + dsOverrideNodes, err := readVarNodes(streamNode) + if err != nil { + return fmt.Errorf("reading DS override var nodes in %q: %w", manifestPath, err) + } + + if err := checkDuplicateVarNodes(dsOverrideNodes); err != nil { + return fmt.Errorf("duplicate vars in data stream manifest %q: %w", manifestPath, err) + } + + mergedSeq, err := mergeStreamLevelVarNodes(baseVarOrder, baseVarByName, promotedNames, dsOverrideNodes) + if err != nil { + return fmt.Errorf("merging stream-level vars in %q: %w", manifestPath, err) + } + + if len(mergedSeq.Content) > 0 { + upsertKey(streamNode, "vars", mergedSeq) + } else { + removeKey(streamNode, "vars") + } + } + + // Step F — Write each updated DS manifest. + dsUpdated, err := formatYAMLNode(&dsDoc) + if err != nil { + return fmt.Errorf("formatting updated data stream manifest %q: %w", manifestPath, err) + } + if err := buildRoot.WriteFile(manifestPath, dsUpdated, 0664); err != nil { + return fmt.Errorf("writing updated data stream manifest %q: %w", manifestPath, err) + } + } + + return nil +} + +// loadInputPkgVarNodes opens the input package at pkgPath, reads all vars from +// all policy templates (dedup by name, first wins) and returns them as an +// ordered slice and a name→node lookup map. +func loadInputPkgVarNodes(pkgPath string) ([]string, map[string]*yaml.Node, error) { + pkgFS, closeFn, err := openPackageFS(pkgPath) + if err != nil { + return nil, nil, fmt.Errorf("opening package: %w", err) + } + defer closeFn() + + manifestBytes, err := fs.ReadFile(pkgFS, packages.PackageManifestFile) + if err != nil { + return nil, nil, fmt.Errorf("reading manifest: %w", err) + } + + var doc yaml.Node + if err := yaml.Unmarshal(manifestBytes, &doc); err != nil { + return nil, nil, fmt.Errorf("parsing manifest YAML: %w", err) + } + + root := &doc + if root.Kind == yaml.DocumentNode { + if len(root.Content) == 0 { + return nil, nil, nil + } + root = root.Content[0] + } + if root.Kind != yaml.MappingNode { + return nil, nil, fmt.Errorf("expected mapping node at document root") + } + + policyTemplatesNode := mappingValue(root, "policy_templates") + if policyTemplatesNode == nil || policyTemplatesNode.Kind != yaml.SequenceNode { + return nil, nil, nil + } + + order := make([]string, 0) + byName := make(map[string]*yaml.Node) + + for _, ptNode := range policyTemplatesNode.Content { + if ptNode.Kind != yaml.MappingNode { + continue + } + varsNode := mappingValue(ptNode, "vars") + if varsNode == nil || varsNode.Kind != yaml.SequenceNode { + continue + } + for _, varNode := range varsNode.Content { + if varNode.Kind != yaml.MappingNode { + continue + } + name := varNodeName(varNode) + if name == "" || byName[name] != nil { + continue // skip empty names and duplicates (first wins) + } + order = append(order, name) + byName[name] = varNode + } + } + + return order, byName, nil +} + +// mergeInputLevelVarNodes returns a sequence node containing only the promoted +// vars (those in promotedOverrides), each merged with the override fields. +// Order follows baseVarOrder (input package declaration order). +func mergeInputLevelVarNodes( + baseVarOrder []string, + baseVarByName map[string]*yaml.Node, + promotedOverrides map[string]*yaml.Node, +) (*yaml.Node, error) { + seqNode := &yaml.Node{Kind: yaml.SequenceNode} + for _, varName := range baseVarOrder { + overrideNode, promoted := promotedOverrides[varName] + if !promoted { + continue + } + merged, err := mergeVarNode(baseVarByName[varName], overrideNode) + if err != nil { + return nil, fmt.Errorf("merging var %q: %w", varName, err) + } + seqNode.Content = append(seqNode.Content, merged) + } + return seqNode, nil +} + +// mergeStreamLevelVarNodes returns a sequence node containing: +// 1. Non-promoted base vars (in input package order), merged with any DS +// override where names match. +// 2. Novel DS vars (names not in baseVarByName) appended in their declaration +// order. +func mergeStreamLevelVarNodes( + baseVarOrder []string, + baseVarByName map[string]*yaml.Node, + promotedNames map[string]bool, + dsOverrides []*yaml.Node, +) (*yaml.Node, error) { + dsOverrideByName := make(map[string]*yaml.Node, len(dsOverrides)) + for _, v := range dsOverrides { + dsOverrideByName[varNodeName(v)] = v + } + + seqNode := &yaml.Node{Kind: yaml.SequenceNode} + + // Non-promoted base vars first (in input pkg order). + for _, varName := range baseVarOrder { + if promotedNames[varName] { + continue + } + baseNode := baseVarByName[varName] + overrideNode, hasOverride := dsOverrideByName[varName] + var ( + merged *yaml.Node + merr error + ) + if hasOverride { + merged, merr = mergeVarNode(baseNode, overrideNode) + } else { + merged = cloneNode(baseNode) + } + if merr != nil { + return nil, fmt.Errorf("merging var %q: %w", varName, merr) + } + seqNode.Content = append(seqNode.Content, merged) + } + + // Novel DS vars (not present in base) appended in declaration order. + for _, v := range dsOverrides { + if _, inBase := baseVarByName[varNodeName(v)]; !inBase { + seqNode.Content = append(seqNode.Content, cloneNode(v)) + } + } + + return seqNode, nil +} + +// mergeVarNode merges fields from overrideNode into a clone of baseNode. +// All keys in override win; absent keys in override are inherited from base. +// The "name" key is always preserved from base. +func mergeVarNode(base, override *yaml.Node) (*yaml.Node, error) { + result := cloneNode(base) + for i := 0; i+1 < len(override.Content); i += 2 { + keyNode := override.Content[i] + valNode := override.Content[i+1] + if keyNode.Value == "name" { + continue // always preserve name from base + } + upsertKey(result, keyNode.Value, cloneNode(valNode)) + } + return result, nil +} + +// checkDuplicateVarNodes returns an error if any var name appears more than +// once in the provided nodes. +func checkDuplicateVarNodes(varNodes []*yaml.Node) error { + seen := make(map[string]bool, len(varNodes)) + for _, v := range varNodes { + name := varNodeName(v) + if seen[name] { + return fmt.Errorf("duplicate variable %q", name) + } + seen[name] = true + } + return nil +} + +// varNodeName extracts the value of the "name" key from a var mapping node. +func varNodeName(v *yaml.Node) string { + nameVal := mappingValue(v, "name") + if nameVal == nil { + return "" + } + return nameVal.Value +} + +// readVarNodes extracts the individual var mapping nodes from the "vars" +// sequence of the given mapping node. Returns nil if no "vars" key is present. +func readVarNodes(mappingNode *yaml.Node) ([]*yaml.Node, error) { + varsNode := mappingValue(mappingNode, "vars") + if varsNode == nil { + return nil, nil + } + if varsNode.Kind != yaml.SequenceNode { + return nil, fmt.Errorf("'vars' is not a sequence node") + } + result := make([]*yaml.Node, 0, len(varsNode.Content)) + for _, item := range varsNode.Content { + if item.Kind != yaml.MappingNode { + return nil, fmt.Errorf("var entry is not a mapping node") + } + result = append(result, item) + } + return result, nil +} + +// getInputMappingNode navigates to policy_templates[ptIdx].inputs[inputIdx] in +// the given YAML document and returns the input mapping node. +func getInputMappingNode(doc *yaml.Node, ptIdx, inputIdx int) (*yaml.Node, error) { + root := doc + if root.Kind == yaml.DocumentNode { + if len(root.Content) == 0 { + return nil, fmt.Errorf("empty YAML document") + } + root = root.Content[0] + } + if root.Kind != yaml.MappingNode { + return nil, fmt.Errorf("expected mapping node at document root") + } + + ptsNode := mappingValue(root, "policy_templates") + if ptsNode == nil || ptsNode.Kind != yaml.SequenceNode { + return nil, fmt.Errorf("'policy_templates' not found or not a sequence") + } + if ptIdx < 0 || ptIdx >= len(ptsNode.Content) { + return nil, fmt.Errorf("policy template index %d out of range (len=%d)", ptIdx, len(ptsNode.Content)) + } + + ptNode := ptsNode.Content[ptIdx] + if ptNode.Kind != yaml.MappingNode { + return nil, fmt.Errorf("policy template %d is not a mapping", ptIdx) + } + + inputsNode := mappingValue(ptNode, "inputs") + if inputsNode == nil || inputsNode.Kind != yaml.SequenceNode { + return nil, fmt.Errorf("'inputs' not found or not a sequence in policy template %d", ptIdx) + } + if inputIdx < 0 || inputIdx >= len(inputsNode.Content) { + return nil, fmt.Errorf("input index %d out of range (len=%d)", inputIdx, len(inputsNode.Content)) + } + + inputNode := inputsNode.Content[inputIdx] + if inputNode.Kind != yaml.MappingNode { + return nil, fmt.Errorf("input %d is not a mapping", inputIdx) + } + + return inputNode, nil +} + +// getStreamMappingNode navigates to streams[streamIdx] in the given YAML +// document and returns the stream mapping node. +func getStreamMappingNode(doc *yaml.Node, streamIdx int) (*yaml.Node, error) { + root := doc + if root.Kind == yaml.DocumentNode { + if len(root.Content) == 0 { + return nil, fmt.Errorf("empty YAML document") + } + root = root.Content[0] + } + if root.Kind != yaml.MappingNode { + return nil, fmt.Errorf("expected mapping node at document root") + } + + streamsNode := mappingValue(root, "streams") + if streamsNode == nil || streamsNode.Kind != yaml.SequenceNode { + return nil, fmt.Errorf("'streams' not found or not a sequence") + } + if streamIdx < 0 || streamIdx >= len(streamsNode.Content) { + return nil, fmt.Errorf("stream index %d out of range (len=%d)", streamIdx, len(streamsNode.Content)) + } + + streamNode := streamsNode.Content[streamIdx] + if streamNode.Kind != yaml.MappingNode { + return nil, fmt.Errorf("stream %d is not a mapping", streamIdx) + } + + return streamNode, nil +} diff --git a/internal/requiredinputs/variables_test.go b/internal/requiredinputs/variables_test.go new file mode 100644 index 0000000000..a21443f34d --- /dev/null +++ b/internal/requiredinputs/variables_test.go @@ -0,0 +1,419 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package requiredinputs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-package/internal/packages" +) + +// ---- helpers ----------------------------------------------------------------- + +// varNode builds a minimal YAML mapping node representing a variable with the +// given name and extra key=value pairs (passed as alternating key, value +// strings for simple scalar values). +func varNode(name string, extras ...string) *yaml.Node { + n := &yaml.Node{Kind: yaml.MappingNode} + upsertKey(n, "name", &yaml.Node{Kind: yaml.ScalarNode, Value: name}) + for i := 0; i+1 < len(extras); i += 2 { + upsertKey(n, extras[i], &yaml.Node{Kind: yaml.ScalarNode, Value: extras[i+1]}) + } + return n +} + +// copyFixturePackage copies the named package from test/packages/required_inputs +// to a fresh temp dir and returns that dir path. +func copyFixturePackage(t *testing.T, fixtureName string) string { + t.Helper() + srcPath := filepath.Join("..", "..", "test", "packages", "required_inputs", fixtureName) + destPath := t.TempDir() + err := os.CopyFS(destPath, os.DirFS(srcPath)) + require.NoError(t, err, "copying fixture package %q", fixtureName) + return destPath +} + +// ---- unit tests -------------------------------------------------------------- + +func TestCloneNode(t *testing.T) { + original := varNode("paths", "type", "text", "multi", "true") + cloned := cloneNode(original) + + // Mutating the clone must not affect the original. + upsertKey(cloned, "type", &yaml.Node{Kind: yaml.ScalarNode, Value: "keyword"}) + assert.Equal(t, "text", mappingValue(original, "type").Value) +} + +func TestMergeVarNode(t *testing.T) { + base := varNode("paths", "type", "text", "title", "Paths", "multi", "true") + + t.Run("full override", func(t *testing.T) { + override := varNode("paths", "type", "keyword", "title", "Custom Paths", "multi", "false") + merged, err := mergeVarNode(base, override) + require.NoError(t, err) + assert.Equal(t, "paths", varNodeName(merged)) + assert.Equal(t, "keyword", mappingValue(merged, "type").Value) + assert.Equal(t, "Custom Paths", mappingValue(merged, "title").Value) + assert.Equal(t, "false", mappingValue(merged, "multi").Value) + }) + + t.Run("partial override", func(t *testing.T) { + override := varNode("paths", "title", "My Paths") + merged, err := mergeVarNode(base, override) + require.NoError(t, err) + assert.Equal(t, "paths", varNodeName(merged)) + assert.Equal(t, "text", mappingValue(merged, "type").Value) // from base + assert.Equal(t, "My Paths", mappingValue(merged, "title").Value) + assert.Equal(t, "true", mappingValue(merged, "multi").Value) // from base + }) + + t.Run("empty override", func(t *testing.T) { + override := varNode("paths") + merged, err := mergeVarNode(base, override) + require.NoError(t, err) + assert.Equal(t, "paths", varNodeName(merged)) + assert.Equal(t, "text", mappingValue(merged, "type").Value) // from base + assert.Equal(t, "Paths", mappingValue(merged, "title").Value) // from base + }) + + t.Run("name not renamed", func(t *testing.T) { + // Even if the override specifies a different name value, base name wins. + override := &yaml.Node{Kind: yaml.MappingNode} + upsertKey(override, "name", &yaml.Node{Kind: yaml.ScalarNode, Value: "should-be-ignored"}) + upsertKey(override, "type", &yaml.Node{Kind: yaml.ScalarNode, Value: "keyword"}) + merged, err := mergeVarNode(base, override) + require.NoError(t, err) + assert.Equal(t, "paths", varNodeName(merged)) + }) + + t.Run("adds new field from override", func(t *testing.T) { + override := varNode("paths", "description", "My description") + merged, err := mergeVarNode(base, override) + require.NoError(t, err) + assert.Equal(t, "My description", mappingValue(merged, "description").Value) + assert.Equal(t, "text", mappingValue(merged, "type").Value) // base preserved + }) +} + +func TestCheckDuplicateVarNodes(t *testing.T) { + t.Run("no duplicates", func(t *testing.T) { + nodes := []*yaml.Node{varNode("paths"), varNode("encoding"), varNode("timeout")} + assert.NoError(t, checkDuplicateVarNodes(nodes)) + }) + + t.Run("one duplicate", func(t *testing.T) { + nodes := []*yaml.Node{varNode("paths"), varNode("encoding"), varNode("paths")} + err := checkDuplicateVarNodes(nodes) + require.Error(t, err) + assert.Contains(t, err.Error(), "paths") + }) + + t.Run("empty slice", func(t *testing.T) { + assert.NoError(t, checkDuplicateVarNodes(nil)) + }) +} + +func TestMergeInputLevelVarNodes(t *testing.T) { + pathsBase := varNode("paths", "type", "text", "multi", "true") + encodingBase := varNode("encoding", "type", "text", "show_user", "false") + timeoutBase := varNode("timeout", "type", "text", "default", "30s") + + baseOrder := []string{"paths", "encoding", "timeout"} + baseByName := map[string]*yaml.Node{ + "paths": pathsBase, + "encoding": encodingBase, + "timeout": timeoutBase, + } + + t.Run("empty promoted → empty sequence", func(t *testing.T) { + seq, err := mergeInputLevelVarNodes(baseOrder, baseByName, map[string]*yaml.Node{}) + require.NoError(t, err) + assert.Empty(t, seq.Content) + }) + + t.Run("one promoted partial override", func(t *testing.T) { + promotedOverrides := map[string]*yaml.Node{ + "paths": varNode("paths", "default", "/var/log/custom/*.log"), + } + seq, err := mergeInputLevelVarNodes(baseOrder, baseByName, promotedOverrides) + require.NoError(t, err) + require.Len(t, seq.Content, 1) + assert.Equal(t, "paths", varNodeName(seq.Content[0])) + assert.Equal(t, "/var/log/custom/*.log", mappingValue(seq.Content[0], "default").Value) + assert.Equal(t, "text", mappingValue(seq.Content[0], "type").Value) // from base + }) + + t.Run("multiple promoted in base order", func(t *testing.T) { + promotedOverrides := map[string]*yaml.Node{ + "timeout": varNode("timeout", "default", "60s"), + "encoding": varNode("encoding", "show_user", "true"), + } + seq, err := mergeInputLevelVarNodes(baseOrder, baseByName, promotedOverrides) + require.NoError(t, err) + require.Len(t, seq.Content, 2) + // Order must follow baseOrder: encoding before timeout. + assert.Equal(t, "encoding", varNodeName(seq.Content[0])) + assert.Equal(t, "timeout", varNodeName(seq.Content[1])) + assert.Equal(t, "true", mappingValue(seq.Content[0], "show_user").Value) + assert.Equal(t, "60s", mappingValue(seq.Content[1], "default").Value) + }) +} + +func TestMergeStreamLevelVarNodes(t *testing.T) { + pathsBase := varNode("paths", "type", "text", "multi", "true") + encodingBase := varNode("encoding", "type", "text", "show_user", "false") + timeoutBase := varNode("timeout", "type", "text", "default", "30s") + + baseOrder := []string{"paths", "encoding", "timeout"} + baseByName := map[string]*yaml.Node{ + "paths": pathsBase, + "encoding": encodingBase, + "timeout": timeoutBase, + } + + t.Run("no promoted, no overrides → all base vars", func(t *testing.T) { + seq, err := mergeStreamLevelVarNodes(baseOrder, baseByName, nil, nil) + require.NoError(t, err) + require.Len(t, seq.Content, 3) + assert.Equal(t, "paths", varNodeName(seq.Content[0])) + assert.Equal(t, "encoding", varNodeName(seq.Content[1])) + assert.Equal(t, "timeout", varNodeName(seq.Content[2])) + }) + + t.Run("some promoted → promoted excluded", func(t *testing.T) { + promoted := map[string]bool{"paths": true, "encoding": true} + seq, err := mergeStreamLevelVarNodes(baseOrder, baseByName, promoted, nil) + require.NoError(t, err) + require.Len(t, seq.Content, 1) + assert.Equal(t, "timeout", varNodeName(seq.Content[0])) + }) + + t.Run("DS override on existing base var", func(t *testing.T) { + dsOverrides := []*yaml.Node{varNode("encoding", "show_user", "true")} + seq, err := mergeStreamLevelVarNodes(baseOrder, baseByName, nil, dsOverrides) + require.NoError(t, err) + require.Len(t, seq.Content, 3) + // encoding is merged + encodingMerged := seq.Content[1] + assert.Equal(t, "encoding", varNodeName(encodingMerged)) + assert.Equal(t, "true", mappingValue(encodingMerged, "show_user").Value) + assert.Equal(t, "text", mappingValue(encodingMerged, "type").Value) // from base + }) + + t.Run("novel DS var appended", func(t *testing.T) { + dsOverrides := []*yaml.Node{varNode("custom_tag", "type", "text")} + seq, err := mergeStreamLevelVarNodes(baseOrder, baseByName, nil, dsOverrides) + require.NoError(t, err) + require.Len(t, seq.Content, 4) // 3 base + 1 novel + assert.Equal(t, "custom_tag", varNodeName(seq.Content[3])) + }) + + t.Run("mixed: promoted + DS merge + novel", func(t *testing.T) { + promoted := map[string]bool{"paths": true} + dsOverrides := []*yaml.Node{ + varNode("encoding", "show_user", "true"), + varNode("custom_tag", "type", "text"), + } + seq, err := mergeStreamLevelVarNodes(baseOrder, baseByName, promoted, dsOverrides) + require.NoError(t, err) + // paths excluded (promoted); encoding merged; timeout base; custom_tag novel + require.Len(t, seq.Content, 3) + assert.Equal(t, "encoding", varNodeName(seq.Content[0])) + assert.Equal(t, "true", mappingValue(seq.Content[0], "show_user").Value) + assert.Equal(t, "timeout", varNodeName(seq.Content[1])) + assert.Equal(t, "custom_tag", varNodeName(seq.Content[2])) + }) +} + +func TestLoadInputPkgVarNodes(t *testing.T) { + t.Run("fixture with three vars", func(t *testing.T) { + pkgPath := filepath.Join("..", "..", "test", "packages", "required_inputs", "var_merging_input_pkg") + order, byName, err := loadInputPkgVarNodes(pkgPath) + require.NoError(t, err) + assert.Equal(t, []string{"paths", "encoding", "timeout"}, order) + assert.Equal(t, "text", mappingValue(byName["paths"], "type").Value) + assert.Equal(t, "text", mappingValue(byName["encoding"], "type").Value) + assert.Equal(t, "text", mappingValue(byName["timeout"], "type").Value) + }) + + t.Run("package with no vars", func(t *testing.T) { + // Use the fake input helper which has no vars in its manifest. + pkgPath := createFakeInputHelper(t) + order, byName, err := loadInputPkgVarNodes(pkgPath) + require.NoError(t, err) + assert.Empty(t, order) + assert.Empty(t, byName) + }) +} + +// ---- integration tests ------------------------------------------------------- + +func makeFakeEprForVarMerging(t *testing.T) *fakeEprClient { + t.Helper() + inputPkgPath := filepath.Join("..", "..", "test", "packages", "required_inputs", "var_merging_input_pkg") + return &fakeEprClient{ + downloadPackageFunc: func(packageName, packageVersion, tmpDir string) (string, error) { + return inputPkgPath, nil + }, + } +} + +func TestMergeVariables_Full(t *testing.T) { + buildPackageRoot := copyFixturePackage(t, "with_merging_full") + resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) + require.NoError(t, err) + + err = resolver.BundleInputPackageTemplates(buildPackageRoot) + require.NoError(t, err) + + // Check package manifest: input should have 2 vars (paths, encoding). + manifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "manifest.yml")) + require.NoError(t, err) + manifest, err := packages.ReadPackageManifestBytes(manifestBytes) + require.NoError(t, err) + + inputVars := manifest.PolicyTemplates[0].Inputs[0].Vars + require.Len(t, inputVars, 2) + assert.Equal(t, "paths", inputVars[0].Name) + assert.Equal(t, "encoding", inputVars[1].Name) + + // paths: base fields preserved, default overridden. + assert.Equal(t, "text", inputVars[0].Type) + require.NotNil(t, inputVars[0].Default) + + // encoding: show_user overridden to true. + assert.True(t, inputVars[1].ShowUser) + assert.Equal(t, "text", inputVars[1].Type) + + // Check DS manifest: streams[0] should have 2 vars (timeout, custom_tag). + dsManifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "data_stream", "var_merging_logs", "manifest.yml")) + require.NoError(t, err) + dsManifest, err := packages.ReadDataStreamManifestBytes(dsManifestBytes) + require.NoError(t, err) + + streamVars := dsManifest.Streams[0].Vars + require.Len(t, streamVars, 2) + assert.Equal(t, "timeout", streamVars[0].Name) + assert.Equal(t, "custom_tag", streamVars[1].Name) + + // timeout: merged from base + DS override (description). + assert.Equal(t, "text", streamVars[0].Type) + assert.Equal(t, "Timeout for log collection.", streamVars[0].Description) +} + +func TestMergeVariables_PromotesToInput(t *testing.T) { + buildPackageRoot := copyFixturePackage(t, "with_merging_promotes_to_input") + resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) + require.NoError(t, err) + + err = resolver.BundleInputPackageTemplates(buildPackageRoot) + require.NoError(t, err) + + manifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "manifest.yml")) + require.NoError(t, err) + manifest, err := packages.ReadPackageManifestBytes(manifestBytes) + require.NoError(t, err) + + // Input should have 1 var: paths (promoted, merged with composable override). + inputVars := manifest.PolicyTemplates[0].Inputs[0].Vars + require.Len(t, inputVars, 1) + assert.Equal(t, "paths", inputVars[0].Name) + assert.Equal(t, "text", inputVars[0].Type) // from base + + // DS should have 2 vars: encoding and timeout (both from base, no DS overrides). + dsManifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "data_stream", "var_merging_logs", "manifest.yml")) + require.NoError(t, err) + dsManifest, err := packages.ReadDataStreamManifestBytes(dsManifestBytes) + require.NoError(t, err) + + streamVars := dsManifest.Streams[0].Vars + require.Len(t, streamVars, 2) + assert.Equal(t, "encoding", streamVars[0].Name) + assert.Equal(t, "timeout", streamVars[1].Name) +} + +func TestMergeVariables_DsMerges(t *testing.T) { + buildPackageRoot := copyFixturePackage(t, "with_merging_ds_merges") + resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) + require.NoError(t, err) + + err = resolver.BundleInputPackageTemplates(buildPackageRoot) + require.NoError(t, err) + + manifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "manifest.yml")) + require.NoError(t, err) + manifest, err := packages.ReadPackageManifestBytes(manifestBytes) + require.NoError(t, err) + + // No input-level vars (nothing promoted). + assert.Empty(t, manifest.PolicyTemplates[0].Inputs[0].Vars) + + // DS should have 4 vars: paths, encoding (merged), timeout, custom_tag. + dsManifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "data_stream", "var_merging_logs", "manifest.yml")) + require.NoError(t, err) + dsManifest, err := packages.ReadDataStreamManifestBytes(dsManifestBytes) + require.NoError(t, err) + + streamVars := dsManifest.Streams[0].Vars + require.Len(t, streamVars, 4) + assert.Equal(t, "paths", streamVars[0].Name) + assert.Equal(t, "encoding", streamVars[1].Name) + assert.Equal(t, "timeout", streamVars[2].Name) + assert.Equal(t, "custom_tag", streamVars[3].Name) + + // encoding: title overridden. + assert.Equal(t, "Log Encoding Override", streamVars[1].Title) + assert.Equal(t, "text", streamVars[1].Type) // from base +} + +func TestMergeVariables_NoOverride(t *testing.T) { + buildPackageRoot := copyFixturePackage(t, "with_merging_no_override") + resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) + require.NoError(t, err) + + err = resolver.BundleInputPackageTemplates(buildPackageRoot) + require.NoError(t, err) + + manifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "manifest.yml")) + require.NoError(t, err) + manifest, err := packages.ReadPackageManifestBytes(manifestBytes) + require.NoError(t, err) + + // No input-level vars. + assert.Empty(t, manifest.PolicyTemplates[0].Inputs[0].Vars) + + // DS should have 3 vars: all from base, unmodified. + dsManifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "data_stream", "var_merging_logs", "manifest.yml")) + require.NoError(t, err) + dsManifest, err := packages.ReadDataStreamManifestBytes(dsManifestBytes) + require.NoError(t, err) + + streamVars := dsManifest.Streams[0].Vars + require.Len(t, streamVars, 3) + assert.Equal(t, "paths", streamVars[0].Name) + assert.Equal(t, "encoding", streamVars[1].Name) + assert.Equal(t, "timeout", streamVars[2].Name) + + // Base fields preserved. + assert.Equal(t, "text", streamVars[0].Type) + assert.True(t, streamVars[0].Multi) + assert.True(t, streamVars[0].Required) +} + +func TestMergeVariables_DuplicateError(t *testing.T) { + buildPackageRoot := copyFixturePackage(t, "with_merging_duplicate_error") + resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) + require.NoError(t, err) + + err = resolver.BundleInputPackageTemplates(buildPackageRoot) + require.Error(t, err) + assert.Contains(t, err.Error(), "paths") +} diff --git a/internal/requiredinputs/yamlutil.go b/internal/requiredinputs/yamlutil.go index e5ddd54110..09aea0aad6 100644 --- a/internal/requiredinputs/yamlutil.go +++ b/internal/requiredinputs/yamlutil.go @@ -48,6 +48,22 @@ func upsertKey(node *yaml.Node, key string, value *yaml.Node) { node.Content = append(node.Content, keyNode, value) } +// cloneNode returns a deep copy of the YAML node tree so base nodes from the +// input package can be reused for multiple independent merges without aliasing. +func cloneNode(n *yaml.Node) *yaml.Node { + if n == nil { + return nil + } + clone := *n + if len(n.Content) > 0 { + clone.Content = make([]*yaml.Node, len(n.Content)) + for i, c := range n.Content { + clone.Content[i] = cloneNode(c) + } + } + return &clone +} + func formatYAMLNode(doc *yaml.Node) ([]byte, error) { raw, err := yaml.Marshal(doc) if err != nil { From 08631a3c6753a295dfabbaded958d1fabd5abd01 Mon Sep 17 00:00:00 2001 From: Tere Date: Tue, 24 Mar 2026 09:17:16 +0100 Subject: [PATCH 3/4] Move var-merge fixtures to manual_packages; document tests - Relocate required_inputs variable-merge packages from test/packages to test/manual_packages/required_inputs (manual-only CI glob). - Point variables_test and loadInputPkgVarNodes fixtures at the new paths. - Refresh test/manual_packages/README for unified required_inputs workflow and future CI move note. - Add with_merging_two_policy_templates for scoped promotion across PTs. - Add mergeVariables-oriented doc comments on variables_test.go. Made-with: Cursor --- internal/requiredinputs/variables_test.go | 118 +++++++++++++++++- test/manual_packages/README.md | 43 +++++-- .../agent/input/input.yml.hbs | 0 .../var_merging_input_pkg/changelog.yml | 0 .../var_merging_input_pkg/docs/README.md | 0 .../fields/base-fields.yml | 0 .../var_merging_input_pkg/manifest.yml | 0 .../_dev/test/config.yml | 0 .../with_merging_ds_merges/changelog.yml | 0 .../agent/stream/stream.yml.hbs | 0 .../var_merging_logs/fields/base-fields.yml | 0 .../data_stream/var_merging_logs/manifest.yml | 0 .../with_merging_ds_merges/docs/README.md | 0 .../with_merging_ds_merges/manifest.yml | 0 .../_dev/test/config.yml | 0 .../changelog.yml | 0 .../agent/stream/stream.yml.hbs | 0 .../var_merging_logs/fields/base-fields.yml | 0 .../data_stream/var_merging_logs/manifest.yml | 0 .../docs/README.md | 0 .../with_merging_duplicate_error/manifest.yml | 0 .../with_merging_full/_dev/test/config.yml | 0 .../with_merging_full/changelog.yml | 0 .../agent/stream/stream.yml.hbs | 0 .../var_merging_logs/fields/base-fields.yml | 0 .../data_stream/var_merging_logs/manifest.yml | 0 .../with_merging_full/docs/README.md | 0 .../with_merging_full/manifest.yml | 0 .../_dev/test/config.yml | 0 .../with_merging_no_override/changelog.yml | 0 .../agent/stream/stream.yml.hbs | 0 .../var_merging_logs/fields/base-fields.yml | 0 .../data_stream/var_merging_logs/manifest.yml | 0 .../with_merging_no_override/docs/README.md | 0 .../with_merging_no_override/manifest.yml | 0 .../_dev/test/config.yml | 0 .../changelog.yml | 0 .../agent/stream/stream.yml.hbs | 0 .../var_merging_logs/fields/base-fields.yml | 0 .../data_stream/var_merging_logs/manifest.yml | 0 .../docs/README.md | 0 .../manifest.yml | 0 .../_dev/test/config.yml | 3 + .../changelog.yml | 5 + .../alpha_logs/agent/stream/stream.yml.hbs | 4 + .../alpha_logs/fields/base-fields.yml | 12 ++ .../data_stream/alpha_logs/manifest.yml | 6 + .../beta_logs/agent/stream/stream.yml.hbs | 4 + .../beta_logs/fields/base-fields.yml | 12 ++ .../data_stream/beta_logs/manifest.yml | 6 + .../docs/README.md | 5 + .../manifest.yml | 52 ++++++++ 52 files changed, 258 insertions(+), 12 deletions(-) rename test/{packages => manual_packages}/required_inputs/var_merging_input_pkg/agent/input/input.yml.hbs (100%) rename test/{packages => manual_packages}/required_inputs/var_merging_input_pkg/changelog.yml (100%) rename test/{packages => manual_packages}/required_inputs/var_merging_input_pkg/docs/README.md (100%) rename test/{packages => manual_packages}/required_inputs/var_merging_input_pkg/fields/base-fields.yml (100%) rename test/{packages => manual_packages}/required_inputs/var_merging_input_pkg/manifest.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_ds_merges/_dev/test/config.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_ds_merges/changelog.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/agent/stream/stream.yml.hbs (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/fields/base-fields.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/manifest.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_ds_merges/docs/README.md (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_ds_merges/manifest.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_duplicate_error/_dev/test/config.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_duplicate_error/changelog.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/agent/stream/stream.yml.hbs (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/fields/base-fields.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/manifest.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_duplicate_error/docs/README.md (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_duplicate_error/manifest.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_full/_dev/test/config.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_full/changelog.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_full/data_stream/var_merging_logs/agent/stream/stream.yml.hbs (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_full/data_stream/var_merging_logs/fields/base-fields.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_full/data_stream/var_merging_logs/manifest.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_full/docs/README.md (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_full/manifest.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_no_override/_dev/test/config.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_no_override/changelog.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_no_override/data_stream/var_merging_logs/agent/stream/stream.yml.hbs (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_no_override/data_stream/var_merging_logs/fields/base-fields.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_no_override/data_stream/var_merging_logs/manifest.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_no_override/docs/README.md (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_no_override/manifest.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_promotes_to_input/_dev/test/config.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_promotes_to_input/changelog.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/agent/stream/stream.yml.hbs (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/fields/base-fields.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/manifest.yml (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_promotes_to_input/docs/README.md (100%) rename test/{packages => manual_packages}/required_inputs/with_merging_promotes_to_input/manifest.yml (100%) create mode 100644 test/manual_packages/required_inputs/with_merging_two_policy_templates/_dev/test/config.yml create mode 100644 test/manual_packages/required_inputs/with_merging_two_policy_templates/changelog.yml create mode 100644 test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/agent/stream/stream.yml.hbs create mode 100644 test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/fields/base-fields.yml create mode 100644 test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/manifest.yml create mode 100644 test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/agent/stream/stream.yml.hbs create mode 100644 test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/fields/base-fields.yml create mode 100644 test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/manifest.yml create mode 100644 test/manual_packages/required_inputs/with_merging_two_policy_templates/docs/README.md create mode 100644 test/manual_packages/required_inputs/with_merging_two_policy_templates/manifest.yml diff --git a/internal/requiredinputs/variables_test.go b/internal/requiredinputs/variables_test.go index a21443f34d..366fb3625a 100644 --- a/internal/requiredinputs/variables_test.go +++ b/internal/requiredinputs/variables_test.go @@ -30,19 +30,30 @@ func varNode(name string, extras ...string) *yaml.Node { return n } -// copyFixturePackage copies the named package from test/packages/required_inputs +// copyFixturePackage copies the named package from test/manual_packages/required_inputs // to a fresh temp dir and returns that dir path. func copyFixturePackage(t *testing.T, fixtureName string) string { t.Helper() - srcPath := filepath.Join("..", "..", "test", "packages", "required_inputs", fixtureName) + srcPath := filepath.Join("..", "..", "test", "manual_packages", "required_inputs", fixtureName) destPath := t.TempDir() err := os.CopyFS(destPath, os.DirFS(srcPath)) require.NoError(t, err, "copying fixture package %q", fixtureName) return destPath } +// Variable merge tests exercise mergeVariables (see variables.go): when an +// integration declares requires.input and references that input package under +// policy_templates[].inputs with optional vars, definitions from the input +// package must be merged into the built integration—composable and data-stream +// overrides on top of the input package as base, with selected vars promoted +// to input-level. Unit tests cover helpers; integration tests run +// BundleInputPackageTemplates on manual fixture packages. + // ---- unit tests -------------------------------------------------------------- +// TestCloneNode checks that YAML variable nodes are deep-cloned before merge. +// mergeVariables mutates cloned trees when applying overrides; without +// isolation, the resolver could corrupt cached or shared input-package nodes. func TestCloneNode(t *testing.T) { original := varNode("paths", "type", "text", "multi", "true") cloned := cloneNode(original) @@ -52,6 +63,11 @@ func TestCloneNode(t *testing.T) { assert.Equal(t, "text", mappingValue(original, "type").Value) } +// TestMergeVarNode verifies mergeVarNode: per-variable field merge where the +// input package definition is the base and override keys from the composable +// package or data stream replace or add fields; the variable name always stays +// from the base. This is the primitive used for both promoted input vars and +// stream-level merges. func TestMergeVarNode(t *testing.T) { base := varNode("paths", "type", "text", "title", "Paths", "multi", "true") @@ -103,6 +119,9 @@ func TestMergeVarNode(t *testing.T) { }) } +// TestCheckDuplicateVarNodes ensures duplicate var names in a single vars list +// are rejected before merge. That catches invalid integration manifests early +// instead of producing ambiguous merged output for Fleet. func TestCheckDuplicateVarNodes(t *testing.T) { t.Run("no duplicates", func(t *testing.T) { nodes := []*yaml.Node{varNode("paths"), varNode("encoding"), varNode("timeout")} @@ -121,6 +140,10 @@ func TestCheckDuplicateVarNodes(t *testing.T) { }) } +// TestMergeInputLevelVarNodes covers mergeInputLevelVarNodes: vars that appear +// under policy_templates[].inputs[] next to package: are promoted +// to merged input-level var definitions, in input-package declaration order, +// with only explicitly listed names included. func TestMergeInputLevelVarNodes(t *testing.T) { pathsBase := varNode("paths", "type", "text", "multi", "true") encodingBase := varNode("encoding", "type", "text", "show_user", "false") @@ -167,6 +190,10 @@ func TestMergeInputLevelVarNodes(t *testing.T) { }) } +// TestMergeStreamLevelVarNodes covers mergeStreamLevelVarNodes: base vars from +// the input package that are not promoted stay on the data stream stream entry; +// they can be field-merged with DS overrides, and DS-only vars are appended. +// Promoted names must not appear on the stream to avoid duplicating Fleet vars. func TestMergeStreamLevelVarNodes(t *testing.T) { pathsBase := varNode("paths", "type", "text", "multi", "true") encodingBase := varNode("encoding", "type", "text", "show_user", "false") @@ -233,9 +260,12 @@ func TestMergeStreamLevelVarNodes(t *testing.T) { }) } +// TestLoadInputPkgVarNodes checks loadInputPkgVarNodes: variable definitions +// are loaded from the resolved input package manifest so mergeVariables uses +// the input package as the authoritative base (order and fields) for merging. func TestLoadInputPkgVarNodes(t *testing.T) { t.Run("fixture with three vars", func(t *testing.T) { - pkgPath := filepath.Join("..", "..", "test", "packages", "required_inputs", "var_merging_input_pkg") + pkgPath := filepath.Join("..", "..", "test", "manual_packages", "required_inputs", "var_merging_input_pkg") order, byName, err := loadInputPkgVarNodes(pkgPath) require.NoError(t, err) assert.Equal(t, []string{"paths", "encoding", "timeout"}, order) @@ -256,9 +286,12 @@ func TestLoadInputPkgVarNodes(t *testing.T) { // ---- integration tests ------------------------------------------------------- +// makeFakeEprForVarMerging supplies the var_merging_input_pkg fixture path as +// if it were downloaded from the registry, so integration tests do not need a +// running stack. func makeFakeEprForVarMerging(t *testing.T) *fakeEprClient { t.Helper() - inputPkgPath := filepath.Join("..", "..", "test", "packages", "required_inputs", "var_merging_input_pkg") + inputPkgPath := filepath.Join("..", "..", "test", "manual_packages", "required_inputs", "var_merging_input_pkg") return &fakeEprClient{ downloadPackageFunc: func(packageName, packageVersion, tmpDir string) (string, error) { return inputPkgPath, nil @@ -266,6 +299,11 @@ func makeFakeEprForVarMerging(t *testing.T) *fakeEprClient { } } +// TestMergeVariables_Full runs the full merge pipeline: composable vars under +// the package input promote paths and encoding to manifest input-level defs +// (merged with input package defaults), while timeout stays on the data stream +// merged with a DS override and a novel DS-only var is appended—matching the +// end state Fleet expects for a mixed promotion + DS customization scenario. func TestMergeVariables_Full(t *testing.T) { buildPackageRoot := copyFixturePackage(t, "with_merging_full") resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) @@ -309,6 +347,10 @@ func TestMergeVariables_Full(t *testing.T) { assert.Equal(t, "Timeout for log collection.", streamVars[0].Description) } +// TestMergeVariables_PromotesToInput verifies partial promotion: only vars +// listed under the composable input move to input level; remaining input +// package vars stay on the stream unchanged when the data stream supplies no +// overrides. func TestMergeVariables_PromotesToInput(t *testing.T) { buildPackageRoot := copyFixturePackage(t, "with_merging_promotes_to_input") resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) @@ -340,6 +382,10 @@ func TestMergeVariables_PromotesToInput(t *testing.T) { assert.Equal(t, "timeout", streamVars[1].Name) } +// TestMergeVariables_DsMerges covers the case where the composable input +// declares no vars (nothing promoted): all base vars remain on the stream, the +// data stream manifest can merge fields into an existing base var (e.g. title), +// and extra stream-only vars are kept in declaration order after base vars. func TestMergeVariables_DsMerges(t *testing.T) { buildPackageRoot := copyFixturePackage(t, "with_merging_ds_merges") resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) @@ -374,6 +420,10 @@ func TestMergeVariables_DsMerges(t *testing.T) { assert.Equal(t, "text", streamVars[1].Type) // from base } +// TestMergeVariables_NoOverride ensures that when the integration does not +// specify composable or data-stream var overrides, merge still materializes +// input package var definitions onto the stream (cloned base) so behavior stays +// correct for packages that only declare requires.input without local var edits. func TestMergeVariables_NoOverride(t *testing.T) { buildPackageRoot := copyFixturePackage(t, "with_merging_no_override") resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) @@ -408,6 +458,9 @@ func TestMergeVariables_NoOverride(t *testing.T) { assert.True(t, streamVars[0].Required) } +// TestMergeVariables_DuplicateError checks that an invalid data stream manifest +// listing the same var name twice fails during mergeVariables, surfacing a +// clear duplicate-variable error instead of silent corruption. func TestMergeVariables_DuplicateError(t *testing.T) { buildPackageRoot := copyFixturePackage(t, "with_merging_duplicate_error") resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) @@ -417,3 +470,60 @@ func TestMergeVariables_DuplicateError(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "paths") } + +// TestMergeVariables_TwoPolicyTemplatesScopedPromotion verifies that promotion +// is scoped per policy template data stream: composable vars under one template +// promote only for that template’s streams; another template referencing the +// same input package without composable vars keeps all base vars on its streams. +// This guards against incorrectly applying one template’s promotions to every +// stream that uses the same input package. +func TestMergeVariables_TwoPolicyTemplatesScopedPromotion(t *testing.T) { + buildPackageRoot := copyFixturePackage(t, "with_merging_two_policy_templates") + resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) + require.NoError(t, err) + + err = resolver.BundleInputPackageTemplates(buildPackageRoot) + require.NoError(t, err) + + manifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "manifest.yml")) + require.NoError(t, err) + manifest, err := packages.ReadPackageManifestBytes(manifestBytes) + require.NoError(t, err) + require.Len(t, manifest.PolicyTemplates, 2) + + // pt_alpha: composable input has promoted paths (merged title). + alphaPT := manifest.PolicyTemplates[0] + require.Equal(t, "pt_alpha", alphaPT.Name) + require.GreaterOrEqual(t, len(alphaPT.Inputs), 1) + alphaInputVars := alphaPT.Inputs[0].Vars + require.Len(t, alphaInputVars, 1) + assert.Equal(t, "paths", alphaInputVars[0].Name) + assert.Equal(t, "Alpha-only promoted paths title", alphaInputVars[0].Title) + assert.Equal(t, "text", alphaInputVars[0].Type) + + // pt_beta: no promotion — no vars on the composable input entry. + betaPT := manifest.PolicyTemplates[1] + require.Equal(t, "pt_beta", betaPT.Name) + assert.Empty(t, betaPT.Inputs[0].Vars) + + // alpha_logs: paths promoted — stream keeps encoding + timeout only. + alphaDSBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "data_stream", "alpha_logs", "manifest.yml")) + require.NoError(t, err) + alphaDS, err := packages.ReadDataStreamManifestBytes(alphaDSBytes) + require.NoError(t, err) + alphaStreamVars := alphaDS.Streams[0].Vars + require.Len(t, alphaStreamVars, 2) + assert.Equal(t, "encoding", alphaStreamVars[0].Name) + assert.Equal(t, "timeout", alphaStreamVars[1].Name) + + // beta_logs: no promotion — all three base vars on the stream. + betaDSBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "data_stream", "beta_logs", "manifest.yml")) + require.NoError(t, err) + betaDS, err := packages.ReadDataStreamManifestBytes(betaDSBytes) + require.NoError(t, err) + betaStreamVars := betaDS.Streams[0].Vars + require.Len(t, betaStreamVars, 3) + assert.Equal(t, "paths", betaStreamVars[0].Name) + assert.Equal(t, "encoding", betaStreamVars[1].Name) + assert.Equal(t, "timeout", betaStreamVars[2].Name) +} diff --git a/test/manual_packages/README.md b/test/manual_packages/README.md index 07fe651b2e..84bdd80dcf 100644 --- a/test/manual_packages/README.md +++ b/test/manual_packages/README.md @@ -1,13 +1,29 @@ # Manual Test Packages -Packages in this directory are **not** picked up by CI build/install scripts (which glob `test/packages/*/*/`). They require manual setup to exercise. +Packages under `test/manual_packages/` are **not** picked up by CI build/install scripts (which glob `test/packages/*/*/`). They require manual setup to exercise. + +All **`requires.input`** and **variable-merge** fixtures live under [`test/manual_packages/required_inputs/`](required_inputs/). The same trees are used as fixtures by `go test` in [`internal/requiredinputs/variables_test.go`](../../internal/requiredinputs/variables_test.go). ## required_inputs -These packages test the `requires.input` (composable input) feature. +### Template bundling (smoke) + +- `required_inputs/test_input_pkg` — input package; install first. +- `required_inputs/with_input_package_requires` — integration that depends on `test_input_pkg`. Minimal composable setup (no `vars` overrides on the package input). + +### Variable merge (composable input vars) -- `required_inputs/test_input_pkg` — the input package that must be installed first. -- `required_inputs/with_input_package_requires` — an integration package that declares a dependency on `test_input_pkg`. +When an integration lists `requires.input` and its policy template references that input package with optional `vars`, elastic-package **merges** variable definitions from the input package into the built manifests (see [`internal/requiredinputs/variables.go`](../../internal/requiredinputs/variables.go) — `mergeVariables`). + +| Package | Role | +| --- | --- | +| `required_inputs/var_merging_input_pkg` | Required input package (`paths`, `encoding`, `timeout`). | +| `required_inputs/with_merging_full` | Promoted `paths` + `encoding`; DS merge for `timeout` + novel `custom_tag`. | +| `required_inputs/with_merging_promotes_to_input` | Only `paths` promoted; DS keeps `encoding`, `timeout`. | +| `required_inputs/with_merging_ds_merges` | No promotion; DS merges `encoding` title + adds `custom_tag`. | +| `required_inputs/with_merging_no_override` | No composable overrides; all base vars on DS, unchanged. | +| `required_inputs/with_merging_two_policy_templates` | Two PTs on the same input pkg: one promotes `paths` for its DS only; the other leaves all vars on the DS (`TestMergeVariables_TwoPolicyTemplatesScopedPromotion`). | +| `required_inputs/with_merging_duplicate_error` | Invalid: duplicate `paths` at DS level; **build should fail** with an error mentioning `paths`. | ### Manual testing workflow @@ -15,14 +31,25 @@ These packages test the `requires.input` (composable input) feature. ```bash elastic-package stack up -d ``` -2. Configure `package_registry.base_url` to point at the stack's registry URL (see `scripts/test-build-install-zip.sh` lines 69–78 for the pattern). -3. Build and install in dependency order: +2. Configure `package_registry.base_url` in `~/.elastic-package/config.yml` so builds can resolve required input packages (see [local package registry how-to](../../docs/howto/local_package_registry.md) and the root [README](../../README.md) `package_registry` section). +3. Build and install in **dependency order** (input packages before integrations that require them). Examples: + + Template bundling smoke: ```bash elastic-package build -C test/manual_packages/required_inputs/test_input_pkg --zip elastic-package build -C test/manual_packages/required_inputs/with_input_package_requires --zip ``` -4. Install via the local registry, `test_input_pkg` first, then `with_input_package_requires`. + + Variable merge (build `var_merging_input_pkg` first, install it, then build the integration you need): + ```bash + elastic-package build -C test/manual_packages/required_inputs/var_merging_input_pkg --zip + elastic-package build -C test/manual_packages/required_inputs/with_merging_full --zip + ``` + +4. Install via the local registry in the same order (e.g. `test_input_pkg` before `with_input_package_requires`; `var_merging_input_pkg` before any `with_merging_*` integration). + +For **expected merged manifests** after a successful variable-merge build, see `TestMergeVariables_*` in [`variables_test.go`](../../internal/requiredinputs/variables_test.go). For `with_merging_duplicate_error`, expect `elastic-package build` to fail and the error to contain `paths`. ### When composable inputs are fully supported in CI -Move `required_inputs/` back to `test/packages/required_inputs/` so the existing install scripts regain automated coverage without requiring additional special-casing. +Move `required_inputs/` under `test/packages/required_inputs/` so [`scripts/test-build-install-zip.sh`](../../scripts/test-build-install-zip.sh) can build and install them automatically (install order is lexicographic, so `var_merging_input_pkg` is installed before `with_merging_*`). Update [`internal/requiredinputs/variables_test.go`](../../internal/requiredinputs/variables_test.go) fixture paths to match. diff --git a/test/packages/required_inputs/var_merging_input_pkg/agent/input/input.yml.hbs b/test/manual_packages/required_inputs/var_merging_input_pkg/agent/input/input.yml.hbs similarity index 100% rename from test/packages/required_inputs/var_merging_input_pkg/agent/input/input.yml.hbs rename to test/manual_packages/required_inputs/var_merging_input_pkg/agent/input/input.yml.hbs diff --git a/test/packages/required_inputs/var_merging_input_pkg/changelog.yml b/test/manual_packages/required_inputs/var_merging_input_pkg/changelog.yml similarity index 100% rename from test/packages/required_inputs/var_merging_input_pkg/changelog.yml rename to test/manual_packages/required_inputs/var_merging_input_pkg/changelog.yml diff --git a/test/packages/required_inputs/var_merging_input_pkg/docs/README.md b/test/manual_packages/required_inputs/var_merging_input_pkg/docs/README.md similarity index 100% rename from test/packages/required_inputs/var_merging_input_pkg/docs/README.md rename to test/manual_packages/required_inputs/var_merging_input_pkg/docs/README.md diff --git a/test/packages/required_inputs/var_merging_input_pkg/fields/base-fields.yml b/test/manual_packages/required_inputs/var_merging_input_pkg/fields/base-fields.yml similarity index 100% rename from test/packages/required_inputs/var_merging_input_pkg/fields/base-fields.yml rename to test/manual_packages/required_inputs/var_merging_input_pkg/fields/base-fields.yml diff --git a/test/packages/required_inputs/var_merging_input_pkg/manifest.yml b/test/manual_packages/required_inputs/var_merging_input_pkg/manifest.yml similarity index 100% rename from test/packages/required_inputs/var_merging_input_pkg/manifest.yml rename to test/manual_packages/required_inputs/var_merging_input_pkg/manifest.yml diff --git a/test/packages/required_inputs/with_merging_ds_merges/_dev/test/config.yml b/test/manual_packages/required_inputs/with_merging_ds_merges/_dev/test/config.yml similarity index 100% rename from test/packages/required_inputs/with_merging_ds_merges/_dev/test/config.yml rename to test/manual_packages/required_inputs/with_merging_ds_merges/_dev/test/config.yml diff --git a/test/packages/required_inputs/with_merging_ds_merges/changelog.yml b/test/manual_packages/required_inputs/with_merging_ds_merges/changelog.yml similarity index 100% rename from test/packages/required_inputs/with_merging_ds_merges/changelog.yml rename to test/manual_packages/required_inputs/with_merging_ds_merges/changelog.yml diff --git a/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/agent/stream/stream.yml.hbs b/test/manual_packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/agent/stream/stream.yml.hbs similarity index 100% rename from test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/agent/stream/stream.yml.hbs rename to test/manual_packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/agent/stream/stream.yml.hbs diff --git a/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/fields/base-fields.yml b/test/manual_packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/fields/base-fields.yml similarity index 100% rename from test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/fields/base-fields.yml rename to test/manual_packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/fields/base-fields.yml diff --git a/test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/manifest.yml b/test/manual_packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/manifest.yml similarity index 100% rename from test/packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/manifest.yml rename to test/manual_packages/required_inputs/with_merging_ds_merges/data_stream/var_merging_logs/manifest.yml diff --git a/test/packages/required_inputs/with_merging_ds_merges/docs/README.md b/test/manual_packages/required_inputs/with_merging_ds_merges/docs/README.md similarity index 100% rename from test/packages/required_inputs/with_merging_ds_merges/docs/README.md rename to test/manual_packages/required_inputs/with_merging_ds_merges/docs/README.md diff --git a/test/packages/required_inputs/with_merging_ds_merges/manifest.yml b/test/manual_packages/required_inputs/with_merging_ds_merges/manifest.yml similarity index 100% rename from test/packages/required_inputs/with_merging_ds_merges/manifest.yml rename to test/manual_packages/required_inputs/with_merging_ds_merges/manifest.yml diff --git a/test/packages/required_inputs/with_merging_duplicate_error/_dev/test/config.yml b/test/manual_packages/required_inputs/with_merging_duplicate_error/_dev/test/config.yml similarity index 100% rename from test/packages/required_inputs/with_merging_duplicate_error/_dev/test/config.yml rename to test/manual_packages/required_inputs/with_merging_duplicate_error/_dev/test/config.yml diff --git a/test/packages/required_inputs/with_merging_duplicate_error/changelog.yml b/test/manual_packages/required_inputs/with_merging_duplicate_error/changelog.yml similarity index 100% rename from test/packages/required_inputs/with_merging_duplicate_error/changelog.yml rename to test/manual_packages/required_inputs/with_merging_duplicate_error/changelog.yml diff --git a/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/agent/stream/stream.yml.hbs b/test/manual_packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/agent/stream/stream.yml.hbs similarity index 100% rename from test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/agent/stream/stream.yml.hbs rename to test/manual_packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/agent/stream/stream.yml.hbs diff --git a/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/fields/base-fields.yml b/test/manual_packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/fields/base-fields.yml similarity index 100% rename from test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/fields/base-fields.yml rename to test/manual_packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/fields/base-fields.yml diff --git a/test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/manifest.yml b/test/manual_packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/manifest.yml similarity index 100% rename from test/packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/manifest.yml rename to test/manual_packages/required_inputs/with_merging_duplicate_error/data_stream/var_merging_logs/manifest.yml diff --git a/test/packages/required_inputs/with_merging_duplicate_error/docs/README.md b/test/manual_packages/required_inputs/with_merging_duplicate_error/docs/README.md similarity index 100% rename from test/packages/required_inputs/with_merging_duplicate_error/docs/README.md rename to test/manual_packages/required_inputs/with_merging_duplicate_error/docs/README.md diff --git a/test/packages/required_inputs/with_merging_duplicate_error/manifest.yml b/test/manual_packages/required_inputs/with_merging_duplicate_error/manifest.yml similarity index 100% rename from test/packages/required_inputs/with_merging_duplicate_error/manifest.yml rename to test/manual_packages/required_inputs/with_merging_duplicate_error/manifest.yml diff --git a/test/packages/required_inputs/with_merging_full/_dev/test/config.yml b/test/manual_packages/required_inputs/with_merging_full/_dev/test/config.yml similarity index 100% rename from test/packages/required_inputs/with_merging_full/_dev/test/config.yml rename to test/manual_packages/required_inputs/with_merging_full/_dev/test/config.yml diff --git a/test/packages/required_inputs/with_merging_full/changelog.yml b/test/manual_packages/required_inputs/with_merging_full/changelog.yml similarity index 100% rename from test/packages/required_inputs/with_merging_full/changelog.yml rename to test/manual_packages/required_inputs/with_merging_full/changelog.yml diff --git a/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/agent/stream/stream.yml.hbs b/test/manual_packages/required_inputs/with_merging_full/data_stream/var_merging_logs/agent/stream/stream.yml.hbs similarity index 100% rename from test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/agent/stream/stream.yml.hbs rename to test/manual_packages/required_inputs/with_merging_full/data_stream/var_merging_logs/agent/stream/stream.yml.hbs diff --git a/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/fields/base-fields.yml b/test/manual_packages/required_inputs/with_merging_full/data_stream/var_merging_logs/fields/base-fields.yml similarity index 100% rename from test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/fields/base-fields.yml rename to test/manual_packages/required_inputs/with_merging_full/data_stream/var_merging_logs/fields/base-fields.yml diff --git a/test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/manifest.yml b/test/manual_packages/required_inputs/with_merging_full/data_stream/var_merging_logs/manifest.yml similarity index 100% rename from test/packages/required_inputs/with_merging_full/data_stream/var_merging_logs/manifest.yml rename to test/manual_packages/required_inputs/with_merging_full/data_stream/var_merging_logs/manifest.yml diff --git a/test/packages/required_inputs/with_merging_full/docs/README.md b/test/manual_packages/required_inputs/with_merging_full/docs/README.md similarity index 100% rename from test/packages/required_inputs/with_merging_full/docs/README.md rename to test/manual_packages/required_inputs/with_merging_full/docs/README.md diff --git a/test/packages/required_inputs/with_merging_full/manifest.yml b/test/manual_packages/required_inputs/with_merging_full/manifest.yml similarity index 100% rename from test/packages/required_inputs/with_merging_full/manifest.yml rename to test/manual_packages/required_inputs/with_merging_full/manifest.yml diff --git a/test/packages/required_inputs/with_merging_no_override/_dev/test/config.yml b/test/manual_packages/required_inputs/with_merging_no_override/_dev/test/config.yml similarity index 100% rename from test/packages/required_inputs/with_merging_no_override/_dev/test/config.yml rename to test/manual_packages/required_inputs/with_merging_no_override/_dev/test/config.yml diff --git a/test/packages/required_inputs/with_merging_no_override/changelog.yml b/test/manual_packages/required_inputs/with_merging_no_override/changelog.yml similarity index 100% rename from test/packages/required_inputs/with_merging_no_override/changelog.yml rename to test/manual_packages/required_inputs/with_merging_no_override/changelog.yml diff --git a/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/agent/stream/stream.yml.hbs b/test/manual_packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/agent/stream/stream.yml.hbs similarity index 100% rename from test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/agent/stream/stream.yml.hbs rename to test/manual_packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/agent/stream/stream.yml.hbs diff --git a/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/fields/base-fields.yml b/test/manual_packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/fields/base-fields.yml similarity index 100% rename from test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/fields/base-fields.yml rename to test/manual_packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/fields/base-fields.yml diff --git a/test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/manifest.yml b/test/manual_packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/manifest.yml similarity index 100% rename from test/packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/manifest.yml rename to test/manual_packages/required_inputs/with_merging_no_override/data_stream/var_merging_logs/manifest.yml diff --git a/test/packages/required_inputs/with_merging_no_override/docs/README.md b/test/manual_packages/required_inputs/with_merging_no_override/docs/README.md similarity index 100% rename from test/packages/required_inputs/with_merging_no_override/docs/README.md rename to test/manual_packages/required_inputs/with_merging_no_override/docs/README.md diff --git a/test/packages/required_inputs/with_merging_no_override/manifest.yml b/test/manual_packages/required_inputs/with_merging_no_override/manifest.yml similarity index 100% rename from test/packages/required_inputs/with_merging_no_override/manifest.yml rename to test/manual_packages/required_inputs/with_merging_no_override/manifest.yml diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/_dev/test/config.yml b/test/manual_packages/required_inputs/with_merging_promotes_to_input/_dev/test/config.yml similarity index 100% rename from test/packages/required_inputs/with_merging_promotes_to_input/_dev/test/config.yml rename to test/manual_packages/required_inputs/with_merging_promotes_to_input/_dev/test/config.yml diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/changelog.yml b/test/manual_packages/required_inputs/with_merging_promotes_to_input/changelog.yml similarity index 100% rename from test/packages/required_inputs/with_merging_promotes_to_input/changelog.yml rename to test/manual_packages/required_inputs/with_merging_promotes_to_input/changelog.yml diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/agent/stream/stream.yml.hbs b/test/manual_packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/agent/stream/stream.yml.hbs similarity index 100% rename from test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/agent/stream/stream.yml.hbs rename to test/manual_packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/agent/stream/stream.yml.hbs diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/fields/base-fields.yml b/test/manual_packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/fields/base-fields.yml similarity index 100% rename from test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/fields/base-fields.yml rename to test/manual_packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/fields/base-fields.yml diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/manifest.yml b/test/manual_packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/manifest.yml similarity index 100% rename from test/packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/manifest.yml rename to test/manual_packages/required_inputs/with_merging_promotes_to_input/data_stream/var_merging_logs/manifest.yml diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/docs/README.md b/test/manual_packages/required_inputs/with_merging_promotes_to_input/docs/README.md similarity index 100% rename from test/packages/required_inputs/with_merging_promotes_to_input/docs/README.md rename to test/manual_packages/required_inputs/with_merging_promotes_to_input/docs/README.md diff --git a/test/packages/required_inputs/with_merging_promotes_to_input/manifest.yml b/test/manual_packages/required_inputs/with_merging_promotes_to_input/manifest.yml similarity index 100% rename from test/packages/required_inputs/with_merging_promotes_to_input/manifest.yml rename to test/manual_packages/required_inputs/with_merging_promotes_to_input/manifest.yml diff --git a/test/manual_packages/required_inputs/with_merging_two_policy_templates/_dev/test/config.yml b/test/manual_packages/required_inputs/with_merging_two_policy_templates/_dev/test/config.yml new file mode 100644 index 0000000000..e958a08627 --- /dev/null +++ b/test/manual_packages/required_inputs/with_merging_two_policy_templates/_dev/test/config.yml @@ -0,0 +1,3 @@ +requires: + - package: var_merging_input_pkg + source: "../../var_merging_input_pkg" diff --git a/test/manual_packages/required_inputs/with_merging_two_policy_templates/changelog.yml b/test/manual_packages/required_inputs/with_merging_two_policy_templates/changelog.yml new file mode 100644 index 0000000000..af392ba551 --- /dev/null +++ b/test/manual_packages/required_inputs/with_merging_two_policy_templates/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.1.0 + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/elastic-package/issues/1 diff --git a/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/agent/stream/stream.yml.hbs b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..9390bc05cb --- /dev/null +++ b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} diff --git a/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/fields/base-fields.yml b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/fields/base-fields.yml new file mode 100644 index 0000000000..0d1791ffed --- /dev/null +++ b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/manifest.yml b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/manifest.yml new file mode 100644 index 0000000000..d1f0fb147d --- /dev/null +++ b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/alpha_logs/manifest.yml @@ -0,0 +1,6 @@ +title: Alpha logs +type: logs +streams: + - package: var_merging_input_pkg + title: Alpha logs via input package + description: Collect alpha logs using the var merging input package. diff --git a/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/agent/stream/stream.yml.hbs b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..9390bc05cb --- /dev/null +++ b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} diff --git a/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/fields/base-fields.yml b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/fields/base-fields.yml new file mode 100644 index 0000000000..0d1791ffed --- /dev/null +++ b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/manifest.yml b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/manifest.yml new file mode 100644 index 0000000000..85d68a6e89 --- /dev/null +++ b/test/manual_packages/required_inputs/with_merging_two_policy_templates/data_stream/beta_logs/manifest.yml @@ -0,0 +1,6 @@ +title: Beta logs +type: logs +streams: + - package: var_merging_input_pkg + title: Beta logs via input package + description: Collect beta logs using the var merging input package. diff --git a/test/manual_packages/required_inputs/with_merging_two_policy_templates/docs/README.md b/test/manual_packages/required_inputs/with_merging_two_policy_templates/docs/README.md new file mode 100644 index 0000000000..a969fc6a65 --- /dev/null +++ b/test/manual_packages/required_inputs/with_merging_two_policy_templates/docs/README.md @@ -0,0 +1,5 @@ +# Variable Merging - Two Policy Templates + +Test fixture: promotion of input-package vars is scoped to the policy +template's `data_streams` list. One template promotes `paths`; the other +leaves all vars on the data stream. diff --git a/test/manual_packages/required_inputs/with_merging_two_policy_templates/manifest.yml b/test/manual_packages/required_inputs/with_merging_two_policy_templates/manifest.yml new file mode 100644 index 0000000000..4b8b7a78e2 --- /dev/null +++ b/test/manual_packages/required_inputs/with_merging_two_policy_templates/manifest.yml @@ -0,0 +1,52 @@ +format_version: 3.6.0 +name: with_merging_two_policy_templates +title: Variable Merging - Two Policy Templates Scoped Promotion +description: >- + Two policy templates share the same required input package: one promotes + "paths" to input-level vars for its data stream only; the other does not + promote any vars, so "paths" stays on the data stream. Exercises per-DS + promotion scoping when multiple templates reference the same input package. +version: 0.1.0 +type: integration +categories: + - custom +conditions: + kibana: + version: "^8.0.0" + elastic: + subscription: basic +requires: + input: + - package: var_merging_input_pkg + version: "0.1.0" +policy_templates: + - name: pt_alpha + title: Alpha logs (paths promoted) + description: Policy template that promotes paths to the composable input + data_streams: + - alpha_logs + inputs: + - package: var_merging_input_pkg + title: Collect via var merging input (alpha) + description: Alpha stream promotes paths to input-level vars + vars: + - name: paths + title: Alpha-only promoted paths title + - type: logs + title: Native logs input (alpha) + description: Fallback logs input for alpha + - name: pt_beta + title: Beta logs (no promotion) + description: Policy template with no composable var overrides + data_streams: + - beta_logs + inputs: + - package: var_merging_input_pkg + title: Collect via var merging input (beta) + description: Beta stream keeps all vars at data-stream level + - type: logs + title: Native logs input (beta) + description: Fallback logs input for beta +owner: + github: elastic/integrations + type: elastic From e03e53168bff9d31fffeeffc793f2dd5f3943c6e Mon Sep 17 00:00:00 2001 From: Tere Date: Thu, 26 Mar 2026 16:08:52 +0100 Subject: [PATCH 4/4] Rename Resolver method to Bundle BundleInputPackageTemplates was too narrow: the resolver path now also merges variables from required input packages and may grow further. Rename the requiredinputs.Resolver method and all call sites, tests, and mocks to Bundle. Made-with: Cursor --- internal/builder/packages.go | 2 +- internal/packages/archetype/package_test.go | 7 +++++-- internal/requiredinputs/requiredinputs.go | 9 +++++---- internal/requiredinputs/requiredinputs_test.go | 16 ++++++++-------- internal/requiredinputs/variables_test.go | 14 +++++++------- internal/resources/fleetpackage_test.go | 7 +++++-- 6 files changed, 31 insertions(+), 24 deletions(-) diff --git a/internal/builder/packages.go b/internal/builder/packages.go index bce6b9dbe9..6244a43701 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -234,7 +234,7 @@ func BuildPackage(options BuildOptions) (string, error) { return "", fmt.Errorf("resolving transform manifests failed: %w", err) } - err = options.RequiredInputsResolver.BundleInputPackageTemplates(buildPackageRoot) + err = options.RequiredInputsResolver.Bundle(buildPackageRoot) if err != nil { return "", fmt.Errorf("bundling input package templates failed: %w", err) } diff --git a/internal/packages/archetype/package_test.go b/internal/packages/archetype/package_test.go index b7761e98f3..216723d492 100644 --- a/internal/packages/archetype/package_test.go +++ b/internal/packages/archetype/package_test.go @@ -97,10 +97,13 @@ func createPackageDescriptorForTest(packageType, kibanaVersion string) PackageDe } type requiredInputsResolverMock struct { - BundleInputPackageTemplatesFunc func(buildPackageRoot string) error + BundleFunc func(buildPackageRoot string) error } -func (r *requiredInputsResolverMock) BundleInputPackageTemplates(buildPackageRoot string) error { +func (r *requiredInputsResolverMock) Bundle(buildPackageRoot string) error { + if r.BundleFunc != nil { + return r.BundleFunc(buildPackageRoot) + } return nil } diff --git a/internal/requiredinputs/requiredinputs.go b/internal/requiredinputs/requiredinputs.go index 1336172195..9d1263fe6b 100644 --- a/internal/requiredinputs/requiredinputs.go +++ b/internal/requiredinputs/requiredinputs.go @@ -20,9 +20,10 @@ type eprClient interface { DownloadPackage(packageName string, packageVersion string, tmpDir string) (string, error) } -// Resolver bundles required input package templates into a built package tree. +// Resolver bundles required input package templates into a built package tree and merges +// variables from those input packages when applicable. type Resolver interface { - BundleInputPackageTemplates(buildPackageRoot string) error + Bundle(buildPackageRoot string) error } // NoopRequiredInputsResolver is a no-op implementation of Resolver. @@ -30,7 +31,7 @@ type Resolver interface { // when implementing local input package resolution for development and testing workflows. type NoopRequiredInputsResolver struct{} -func (r *NoopRequiredInputsResolver) BundleInputPackageTemplates(_ string) error { +func (r *NoopRequiredInputsResolver) Bundle(_ string) error { return nil } @@ -46,7 +47,7 @@ func NewRequiredInputsResolver(eprClient eprClient) (*RequiredInputsResolver, er }, nil } -func (r *RequiredInputsResolver) BundleInputPackageTemplates(buildPackageRoot string) error { +func (r *RequiredInputsResolver) Bundle(buildPackageRoot string) error { buildRoot, err := os.OpenRoot(buildPackageRoot) if err != nil { diff --git a/internal/requiredinputs/requiredinputs_test.go b/internal/requiredinputs/requiredinputs_test.go index 5ee5ddaa49..643b3ab63c 100644 --- a/internal/requiredinputs/requiredinputs_test.go +++ b/internal/requiredinputs/requiredinputs_test.go @@ -27,7 +27,7 @@ func (f *fakeEprClient) DownloadPackage(packageName string, packageVersion strin return "", fmt.Errorf("download package not implemented") } -func TestBundleInputPackageTemplates_Success(t *testing.T) { +func TestBundle_Success(t *testing.T) { fakeInputPath := createFakeInputHelper(t) fakeEprClient := &fakeEprClient{ downloadPackageFunc: func(packageName string, packageVersion string, tmpDir string) (string, error) { @@ -54,7 +54,7 @@ policy_templates: resolver, err := NewRequiredInputsResolver(fakeEprClient) require.NoError(t, err) - err = resolver.BundleInputPackageTemplates(buildPackageRoot) + err = resolver.Bundle(buildPackageRoot) require.NoError(t, err) _, err = os.ReadFile(path.Join(buildPackageRoot, "agent", "input", "sql-input.yml.hbs")) @@ -74,7 +74,7 @@ policy_templates: } -func TestBundleInputPackageTemplates_NoManifest(t *testing.T) { +func TestBundle_NoManifest(t *testing.T) { fakeInputPath := createFakeInputHelper(t) fakeEprClient := &fakeEprClient{ downloadPackageFunc: func(packageName string, packageVersion string, tmpDir string) (string, error) { @@ -86,12 +86,12 @@ func TestBundleInputPackageTemplates_NoManifest(t *testing.T) { resolver, err := NewRequiredInputsResolver(fakeEprClient) require.NoError(t, err) - err = resolver.BundleInputPackageTemplates(buildPackageRoot) + err = resolver.Bundle(buildPackageRoot) require.Error(t, err) assert.ErrorContains(t, err, "failed to read package manifest") } -func TestBundleInputPackageTemplates_SkipNoIntegration(t *testing.T) { +func TestBundle_SkipNoIntegration(t *testing.T) { fakeInputPath := createFakeInputHelper(t) fakeEprClient := &fakeEprClient{ downloadPackageFunc: func(packageName string, packageVersion string, tmpDir string) (string, error) { @@ -110,11 +110,11 @@ type: input resolver, err := NewRequiredInputsResolver(fakeEprClient) require.NoError(t, err) - err = resolver.BundleInputPackageTemplates(buildPackageRoot) + err = resolver.Bundle(buildPackageRoot) require.NoError(t, err) } -func TestBundleInputPackageTemplates_NoRequires(t *testing.T) { +func TestBundle_NoRequires(t *testing.T) { fakeEprClient := &fakeEprClient{ downloadPackageFunc: func(packageName string, packageVersion string, tmpDir string) (string, error) { return "", fmt.Errorf("no download without requires") @@ -135,7 +135,7 @@ policy_templates: resolver, err := NewRequiredInputsResolver(fakeEprClient) require.NoError(t, err) - err = resolver.BundleInputPackageTemplates(buildPackageRoot) + err = resolver.Bundle(buildPackageRoot) require.NoError(t, err) updatedManifestBytes, err := os.ReadFile(path.Join(buildPackageRoot, "manifest.yml")) diff --git a/internal/requiredinputs/variables_test.go b/internal/requiredinputs/variables_test.go index 366fb3625a..2394aebad5 100644 --- a/internal/requiredinputs/variables_test.go +++ b/internal/requiredinputs/variables_test.go @@ -47,7 +47,7 @@ func copyFixturePackage(t *testing.T, fixtureName string) string { // package must be merged into the built integration—composable and data-stream // overrides on top of the input package as base, with selected vars promoted // to input-level. Unit tests cover helpers; integration tests run -// BundleInputPackageTemplates on manual fixture packages. +// Integration tests exercise Bundle on manual fixture packages. // ---- unit tests -------------------------------------------------------------- @@ -309,7 +309,7 @@ func TestMergeVariables_Full(t *testing.T) { resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) require.NoError(t, err) - err = resolver.BundleInputPackageTemplates(buildPackageRoot) + err = resolver.Bundle(buildPackageRoot) require.NoError(t, err) // Check package manifest: input should have 2 vars (paths, encoding). @@ -356,7 +356,7 @@ func TestMergeVariables_PromotesToInput(t *testing.T) { resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) require.NoError(t, err) - err = resolver.BundleInputPackageTemplates(buildPackageRoot) + err = resolver.Bundle(buildPackageRoot) require.NoError(t, err) manifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "manifest.yml")) @@ -391,7 +391,7 @@ func TestMergeVariables_DsMerges(t *testing.T) { resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) require.NoError(t, err) - err = resolver.BundleInputPackageTemplates(buildPackageRoot) + err = resolver.Bundle(buildPackageRoot) require.NoError(t, err) manifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "manifest.yml")) @@ -429,7 +429,7 @@ func TestMergeVariables_NoOverride(t *testing.T) { resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) require.NoError(t, err) - err = resolver.BundleInputPackageTemplates(buildPackageRoot) + err = resolver.Bundle(buildPackageRoot) require.NoError(t, err) manifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "manifest.yml")) @@ -466,7 +466,7 @@ func TestMergeVariables_DuplicateError(t *testing.T) { resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) require.NoError(t, err) - err = resolver.BundleInputPackageTemplates(buildPackageRoot) + err = resolver.Bundle(buildPackageRoot) require.Error(t, err) assert.Contains(t, err.Error(), "paths") } @@ -482,7 +482,7 @@ func TestMergeVariables_TwoPolicyTemplatesScopedPromotion(t *testing.T) { resolver, err := NewRequiredInputsResolver(makeFakeEprForVarMerging(t)) require.NoError(t, err) - err = resolver.BundleInputPackageTemplates(buildPackageRoot) + err = resolver.Bundle(buildPackageRoot) require.NoError(t, err) manifestBytes, err := os.ReadFile(filepath.Join(buildPackageRoot, "manifest.yml")) diff --git a/internal/resources/fleetpackage_test.go b/internal/resources/fleetpackage_test.go index bdcd5b1e50..d4036dd400 100644 --- a/internal/resources/fleetpackage_test.go +++ b/internal/resources/fleetpackage_test.go @@ -40,10 +40,13 @@ func TestRequiredProvider(t *testing.T) { } type requiredInputsResolverMock struct { - BundleInputPackageTemplatesFunc func(buildPackageRoot string) error + BundleFunc func(buildPackageRoot string) error } -func (r *requiredInputsResolverMock) BundleInputPackageTemplates(buildPackageRoot string) error { +func (r *requiredInputsResolverMock) Bundle(buildPackageRoot string) error { + if r.BundleFunc != nil { + return r.BundleFunc(buildPackageRoot) + } return nil }