Fix template path validator for composable integration packages

code/go/internal/validator/semantic/validate_integration_policy_template_path.go

@@ -23,6 +23,7 @@ const (
 
 type policyTemplateInput struct {
 	Type         string `yaml:"type"`
+	Package      string `yaml:"package"`       // alternative to type for composable packages
 	TemplatePath string `yaml:"template_path"` // optional for integration packages
 }
 
@@ -38,6 +39,7 @@ type integrationPackageManifest struct { // package manifest
 
 type stream struct {
 	Input        string `yaml:"input"`
+	Package      string `yaml:"package"` // alternative to input for composable packages
 	TemplatePath string `yaml:"template_path"`
 }
 
@@ -90,6 +92,12 @@ func ValidateIntegrationPolicyTemplates(fsys fspath.FS) specerrors.ValidationErr
 // validateIntegrationPackagePolicyTemplate validates the template_path fields at the policy template level for integration type packages
 func validateIntegrationPackagePolicyTemplate(fsys fspath.FS, policyTemplate integrationPolicyTemplate, dsManifestMap map[string]dataStreamManifest) error {
 	for _, input := range policyTemplate.Inputs {
+		// inputs using package references inherit configuration from the input package,
+		// so there is no local agent stream template to validate.
+		if input.Package != "" {
+			continue
+		}
+
 		if input.TemplatePath != "" {
 			// validate the provided template_path file exists
 			err := validateAgentInputTemplatePath(fsys, input.TemplatePath)
@@ -139,6 +147,10 @@ func readDataStreamsManifests(fsys fspath.FS) (map[string]dataStreamManifest, er
 func validateInputWithStreams(fsys fspath.FS, input string, dsMap map[string]dataStreamManifest) error {
 	for dsDir, manifest := range dsMap {
 		for _, stream := range manifest.Streams {
+			// skip streams that use a package reference; they have no local agent stream template
+			if stream.Package != "" {
+				continue
+			}
 			// only consider streams that match the input type of the policy template
 			if stream.Input != input {
 				continue

code/go/pkg/validator/validator_test.go

@@ -47,6 +47,9 @@ func TestValidateFile(t *testing.T) {
 		"good_alert_rule_templates":              {},
 		"good_requires":                          {},
 		"good_package_reference_policy_template": {},
+		"good_composable_pt_inputs":              {},
+		"good_composable_ds_streams":             {},
+		"good_composable_multi_package":          {},
 		"deploy_custom_agent":                    {},
 		"deploy_custom_agent_multi_services":     {},
 		"deploy_docker":                          {},

spec/changelog.yml

@@ -52,6 +52,9 @@
     - description: Add support for `profiles` policy template type in input packages
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/1092
+    - description: Fix template path validator to skip policy template inputs and data stream streams that use package references.
+      type: bugfix
+      link: https://github.com/elastic/package-spec/pull/1108
 - version: 3.5.8
   changes:
     - description: Add deployer option to system benchmark configuration.

test/packages/good_composable_ds_streams/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.0.1
+  changes:
+    - description: Initial release
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/1

test/packages/good_composable_ds_streams/data_stream/logs/fields/fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/good_composable_ds_streams/data_stream/logs/manifest.yml

@@ -0,0 +1,6 @@
+title: Web server logs
+type: logs
+streams:
+  - package: filelog_otel
+    title: Web Server Access Logs
+    description: Collect web server access logs using the filelog OTel input package

test/packages/good_composable_ds_streams/docs/README.md

@@ -0,0 +1,3 @@
+# Composable Package - Data Stream Streams
+
+This is a test package to validate that all data stream streams can reference an input package via the `package` field instead of the traditional `input` field. The `filelog_otel` package is declared in `requires.input` and referenced directly in the data stream stream entry.

test/packages/good_composable_ds_streams/manifest.yml

@@ -0,0 +1,26 @@
+format_version: 3.6.0
+name: good_composable_ds_streams
+title: Composable Package - Data Stream Streams
+description: Integration package with all data stream streams using package references.
+version: 0.0.1
+type: integration
+source:
+  license: "Apache-2.0"
+conditions:
+  kibana:
+    version: '^8.0.0'
+requires:
+  input:
+    - package: filelog_otel
+      version: "1.0.0"
+policy_templates:
+  - name: webserver
+    title: Web server monitoring
+    description: Collect logs from web servers
+    inputs:
+      - type: logfile
+        title: Collect web server logs
+        description: Collecting web server logs using logfile
+owner:
+  github: elastic/foobar
+  type: community

test/packages/good_composable_multi_package/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.0.1
+  changes:
+    - description: Initial release
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/1

test/packages/good_composable_multi_package/data_stream/db_metrics/fields/fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/good_composable_multi_package/data_stream/db_metrics/manifest.yml

@@ -0,0 +1,6 @@
+title: Database metrics
+type: metrics
+streams:
+  - package: sql_input
+    title: Database Metrics
+    description: Collect database metrics using the SQL input package

test/packages/good_composable_multi_package/data_stream/web_logs/fields/fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/good_composable_multi_package/data_stream/web_logs/manifest.yml

@@ -0,0 +1,6 @@
+title: Web server logs
+type: logs
+streams:
+  - package: filelog_otel
+    title: Web Server Logs
+    description: Collect web server logs using the filelog OTel input package

test/packages/good_composable_multi_package/docs/README.md

@@ -0,0 +1,3 @@
+# Composable Package - Multiple Input Packages
+
+This is a test package to validate that multiple input packages can be declared in `requires.input` and referenced via the `package` field in both policy template inputs and data stream streams. It also demonstrates `requires.content` for content package dependencies, which accept version constraints and cannot be referenced via `package`.

test/packages/good_composable_multi_package/manifest.yml

@@ -0,0 +1,40 @@
+format_version: 3.6.0
+name: good_composable_multi_package
+title: Composable Package - Multiple Input Packages
+description: Integration using multiple input packages in both policy templates and data streams, with content package dependencies.
+version: 0.0.1
+type: integration
+source:
+  license: "Apache-2.0"
+conditions:
+  kibana:
+    version: '^8.0.0'
+requires:
+  input:
+    - package: sql_input
+      version: "1.0.0"
+    - package: filelog_otel
+      version: "2.0.0"
+  content:
+    - package: elastic_agent
+      version: "^1.0.0"
+    - package: system
+      version: "~2.0.0"
+policy_templates:
+  - name: database
+    title: Database monitoring
+    description: Collect metrics from databases
+    inputs:
+      - package: sql_input
+        title: Collect database metrics
+        description: Collecting database metrics using the SQL input package
+  - name: webserver
+    title: Web server monitoring
+    description: Collect logs from web servers
+    inputs:
+      - package: filelog_otel
+        title: Collect web server logs
+        description: Collecting web server logs using the filelog OTel input package
+owner:
+  github: elastic/foobar
+  type: community

test/packages/good_composable_pt_inputs/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.0.1
+  changes:
+    - description: Initial release
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/1

test/packages/good_composable_pt_inputs/data_stream/metrics/agent/stream/stream.yml.hbs

@@ -0,0 +1,8 @@
+data_stream:
+  {{#if data_stream.dataset}}
+  dataset: {{data_stream.dataset}}
+  {{/if}}
+paths:
+{{#each paths}}
+  - {{this}}
+{{/each}}

test/packages/good_composable_pt_inputs/data_stream/metrics/fields/fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/good_composable_pt_inputs/data_stream/metrics/manifest.yml

@@ -0,0 +1,6 @@
+title: Database metrics
+type: metrics
+streams:
+  - input: logfile
+    title: Database Metrics
+    description: Collect database metrics using logfile input

test/packages/good_composable_pt_inputs/docs/README.md

@@ -0,0 +1,3 @@
+# Composable Package - Policy Template Inputs
+
+This is a test package to validate that all policy template inputs can reference an input package via the `package` field instead of the traditional `type` field. The `sql_input` package is declared in `requires.input` and referenced directly in the policy template input.

test/packages/good_composable_pt_inputs/manifest.yml

@@ -0,0 +1,26 @@
+format_version: 3.6.0
+name: good_composable_pt_inputs
+title: Composable Package - Policy Template Inputs
+description: Integration package with all policy template inputs using package references.
+version: 0.0.1
+type: integration
+source:
+  license: "Apache-2.0"
+conditions:
+  kibana:
+    version: '^8.0.0'
+requires:
+  input:
+    - package: sql_input
+      version: "1.0.0"
+policy_templates:
+  - name: database
+    title: Database monitoring
+    description: Collect metrics and logs from databases
+    inputs:
+      - package: sql_input
+        title: Collect database metrics
+        description: Collecting database metrics using the SQL input package
+owner:
+  github: elastic/foobar
+  type: community