diff --git a/code/go/internal/validator/semantic/validate_integration_policy_template_path.go b/code/go/internal/validator/semantic/validate_integration_policy_template_path.go index 745bb8ac0..ed8f33ff9 100644 --- a/code/go/internal/validator/semantic/validate_integration_policy_template_path.go +++ b/code/go/internal/validator/semantic/validate_integration_policy_template_path.go @@ -23,6 +23,7 @@ const ( type policyTemplateInput struct { Type string `yaml:"type"` + Package string `yaml:"package"` // alternative to type for composable packages TemplatePath string `yaml:"template_path"` // optional for integration packages } @@ -38,6 +39,7 @@ type integrationPackageManifest struct { // package manifest type stream struct { Input string `yaml:"input"` + Package string `yaml:"package"` // alternative to input for composable packages TemplatePath string `yaml:"template_path"` } @@ -90,6 +92,12 @@ func ValidateIntegrationPolicyTemplates(fsys fspath.FS) specerrors.ValidationErr // validateIntegrationPackagePolicyTemplate validates the template_path fields at the policy template level for integration type packages func validateIntegrationPackagePolicyTemplate(fsys fspath.FS, policyTemplate integrationPolicyTemplate, dsManifestMap map[string]dataStreamManifest) error { for _, input := range policyTemplate.Inputs { + // inputs using package references inherit configuration from the input package, + // so there is no local agent stream template to validate. + if input.Package != "" { + continue + } + if input.TemplatePath != "" { // validate the provided template_path file exists err := validateAgentInputTemplatePath(fsys, input.TemplatePath) @@ -139,6 +147,10 @@ func readDataStreamsManifests(fsys fspath.FS) (map[string]dataStreamManifest, er func validateInputWithStreams(fsys fspath.FS, input string, dsMap map[string]dataStreamManifest) error { for dsDir, manifest := range dsMap { for _, stream := range manifest.Streams { + // skip streams that use a package reference; they have no local agent stream template + if stream.Package != "" { + continue + } // only consider streams that match the input type of the policy template if stream.Input != input { continue diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index 518212d6c..84a0d118c 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -47,6 +47,9 @@ func TestValidateFile(t *testing.T) { "good_alert_rule_templates": {}, "good_requires": {}, "good_package_reference_policy_template": {}, + "good_composable_pt_inputs": {}, + "good_composable_ds_streams": {}, + "good_composable_multi_package": {}, "deploy_custom_agent": {}, "deploy_custom_agent_multi_services": {}, "deploy_docker": {}, diff --git a/spec/changelog.yml b/spec/changelog.yml index 4a437739d..856a459aa 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -52,6 +52,9 @@ - description: Add support for `profiles` policy template type in input packages type: enhancement link: https://github.com/elastic/package-spec/pull/1092 + - description: Fix template path validator to skip policy template inputs and data stream streams that use package references. + type: bugfix + link: https://github.com/elastic/package-spec/pull/1108 - version: 3.5.8 changes: - description: Add deployer option to system benchmark configuration. diff --git a/test/packages/good_composable_ds_streams/changelog.yml b/test/packages/good_composable_ds_streams/changelog.yml new file mode 100644 index 000000000..27e62eecd --- /dev/null +++ b/test/packages/good_composable_ds_streams/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/1 diff --git a/test/packages/good_composable_ds_streams/data_stream/logs/fields/fields.yml b/test/packages/good_composable_ds_streams/data_stream/logs/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/good_composable_ds_streams/data_stream/logs/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_composable_ds_streams/data_stream/logs/manifest.yml b/test/packages/good_composable_ds_streams/data_stream/logs/manifest.yml new file mode 100644 index 000000000..d10226c8f --- /dev/null +++ b/test/packages/good_composable_ds_streams/data_stream/logs/manifest.yml @@ -0,0 +1,6 @@ +title: Web server logs +type: logs +streams: + - package: filelog_otel + title: Web Server Access Logs + description: Collect web server access logs using the filelog OTel input package diff --git a/test/packages/good_composable_ds_streams/docs/README.md b/test/packages/good_composable_ds_streams/docs/README.md new file mode 100644 index 000000000..33db772d9 --- /dev/null +++ b/test/packages/good_composable_ds_streams/docs/README.md @@ -0,0 +1,3 @@ +# Composable Package - Data Stream Streams + +This is a test package to validate that all data stream streams can reference an input package via the `package` field instead of the traditional `input` field. The `filelog_otel` package is declared in `requires.input` and referenced directly in the data stream stream entry. diff --git a/test/packages/good_composable_ds_streams/manifest.yml b/test/packages/good_composable_ds_streams/manifest.yml new file mode 100644 index 000000000..ba7321d27 --- /dev/null +++ b/test/packages/good_composable_ds_streams/manifest.yml @@ -0,0 +1,26 @@ +format_version: 3.6.0 +name: good_composable_ds_streams +title: Composable Package - Data Stream Streams +description: Integration package with all data stream streams using package references. +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0' +requires: + input: + - package: filelog_otel + version: "1.0.0" +policy_templates: + - name: webserver + title: Web server monitoring + description: Collect logs from web servers + inputs: + - type: logfile + title: Collect web server logs + description: Collecting web server logs using logfile +owner: + github: elastic/foobar + type: community diff --git a/test/packages/good_composable_multi_package/changelog.yml b/test/packages/good_composable_multi_package/changelog.yml new file mode 100644 index 000000000..27e62eecd --- /dev/null +++ b/test/packages/good_composable_multi_package/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/1 diff --git a/test/packages/good_composable_multi_package/data_stream/db_metrics/fields/fields.yml b/test/packages/good_composable_multi_package/data_stream/db_metrics/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/good_composable_multi_package/data_stream/db_metrics/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_composable_multi_package/data_stream/db_metrics/manifest.yml b/test/packages/good_composable_multi_package/data_stream/db_metrics/manifest.yml new file mode 100644 index 000000000..d7b2065ee --- /dev/null +++ b/test/packages/good_composable_multi_package/data_stream/db_metrics/manifest.yml @@ -0,0 +1,6 @@ +title: Database metrics +type: metrics +streams: + - package: sql_input + title: Database Metrics + description: Collect database metrics using the SQL input package diff --git a/test/packages/good_composable_multi_package/data_stream/web_logs/fields/fields.yml b/test/packages/good_composable_multi_package/data_stream/web_logs/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/good_composable_multi_package/data_stream/web_logs/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_composable_multi_package/data_stream/web_logs/manifest.yml b/test/packages/good_composable_multi_package/data_stream/web_logs/manifest.yml new file mode 100644 index 000000000..6a9899c62 --- /dev/null +++ b/test/packages/good_composable_multi_package/data_stream/web_logs/manifest.yml @@ -0,0 +1,6 @@ +title: Web server logs +type: logs +streams: + - package: filelog_otel + title: Web Server Logs + description: Collect web server logs using the filelog OTel input package diff --git a/test/packages/good_composable_multi_package/docs/README.md b/test/packages/good_composable_multi_package/docs/README.md new file mode 100644 index 000000000..cb430492f --- /dev/null +++ b/test/packages/good_composable_multi_package/docs/README.md @@ -0,0 +1,3 @@ +# Composable Package - Multiple Input Packages + +This is a test package to validate that multiple input packages can be declared in `requires.input` and referenced via the `package` field in both policy template inputs and data stream streams. It also demonstrates `requires.content` for content package dependencies, which accept version constraints and cannot be referenced via `package`. diff --git a/test/packages/good_composable_multi_package/manifest.yml b/test/packages/good_composable_multi_package/manifest.yml new file mode 100644 index 000000000..e4413de78 --- /dev/null +++ b/test/packages/good_composable_multi_package/manifest.yml @@ -0,0 +1,40 @@ +format_version: 3.6.0 +name: good_composable_multi_package +title: Composable Package - Multiple Input Packages +description: Integration using multiple input packages in both policy templates and data streams, with content package dependencies. +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0' +requires: + input: + - package: sql_input + version: "1.0.0" + - package: filelog_otel + version: "2.0.0" + content: + - package: elastic_agent + version: "^1.0.0" + - package: system + version: "~2.0.0" +policy_templates: + - name: database + title: Database monitoring + description: Collect metrics from databases + inputs: + - package: sql_input + title: Collect database metrics + description: Collecting database metrics using the SQL input package + - name: webserver + title: Web server monitoring + description: Collect logs from web servers + inputs: + - package: filelog_otel + title: Collect web server logs + description: Collecting web server logs using the filelog OTel input package +owner: + github: elastic/foobar + type: community diff --git a/test/packages/good_composable_pt_inputs/changelog.yml b/test/packages/good_composable_pt_inputs/changelog.yml new file mode 100644 index 000000000..27e62eecd --- /dev/null +++ b/test/packages/good_composable_pt_inputs/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/1 diff --git a/test/packages/good_composable_pt_inputs/data_stream/metrics/agent/stream/stream.yml.hbs b/test/packages/good_composable_pt_inputs/data_stream/metrics/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..299dab52c --- /dev/null +++ b/test/packages/good_composable_pt_inputs/data_stream/metrics/agent/stream/stream.yml.hbs @@ -0,0 +1,8 @@ +data_stream: + {{#if data_stream.dataset}} + dataset: {{data_stream.dataset}} + {{/if}} +paths: +{{#each paths}} + - {{this}} +{{/each}} diff --git a/test/packages/good_composable_pt_inputs/data_stream/metrics/fields/fields.yml b/test/packages/good_composable_pt_inputs/data_stream/metrics/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/good_composable_pt_inputs/data_stream/metrics/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_composable_pt_inputs/data_stream/metrics/manifest.yml b/test/packages/good_composable_pt_inputs/data_stream/metrics/manifest.yml new file mode 100644 index 000000000..d32c47726 --- /dev/null +++ b/test/packages/good_composable_pt_inputs/data_stream/metrics/manifest.yml @@ -0,0 +1,6 @@ +title: Database metrics +type: metrics +streams: + - input: logfile + title: Database Metrics + description: Collect database metrics using logfile input diff --git a/test/packages/good_composable_pt_inputs/docs/README.md b/test/packages/good_composable_pt_inputs/docs/README.md new file mode 100644 index 000000000..dfc92ff1e --- /dev/null +++ b/test/packages/good_composable_pt_inputs/docs/README.md @@ -0,0 +1,3 @@ +# Composable Package - Policy Template Inputs + +This is a test package to validate that all policy template inputs can reference an input package via the `package` field instead of the traditional `type` field. The `sql_input` package is declared in `requires.input` and referenced directly in the policy template input. diff --git a/test/packages/good_composable_pt_inputs/manifest.yml b/test/packages/good_composable_pt_inputs/manifest.yml new file mode 100644 index 000000000..7eab17b6f --- /dev/null +++ b/test/packages/good_composable_pt_inputs/manifest.yml @@ -0,0 +1,26 @@ +format_version: 3.6.0 +name: good_composable_pt_inputs +title: Composable Package - Policy Template Inputs +description: Integration package with all policy template inputs using package references. +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0' +requires: + input: + - package: sql_input + version: "1.0.0" +policy_templates: + - name: database + title: Database monitoring + description: Collect metrics and logs from databases + inputs: + - package: sql_input + title: Collect database metrics + description: Collecting database metrics using the SQL input package +owner: + github: elastic/foobar + type: community