Support building integration packages with required input dependencies

README.md

@@ -238,6 +238,8 @@ Built packages are served up by the Elastic Package Registry running locally (se
 
 Built packages can also be published to the global package registry service.
 
+When the package declares required input packages ("requires.input" in manifest.yml), the build downloads those input packages from the configured package registry (see "package_registry.base_url" in ~/.elastic-package/config.yml). The build then incorporates their policy and data stream templates, merges variable definitions into the integration manifest, bundles data stream field definitions, and resolves package: references on inputs and streams to the effective input types expected by Fleet. For details on using a local or custom registry during development, see the [HOWTO guide](./docs/howto/local_package_registry.md).
+
 For details on how to enable dependency management, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/dependency_management.md).
 
 ### `elastic-package changelog`

cmd/benchmark.go

@@ -29,6 +29,8 @@ import (
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
 	"github.com/elastic/elastic-package/internal/signal"
 	"github.com/elastic/elastic-package/internal/stack"
 	"github.com/elastic/elastic-package/internal/testrunner"
@@ -331,6 +333,18 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't create Kibana client: %w", err)
 	}
 
+	appConfig, err := install.Configuration()
+	if err != nil {
+		return fmt.Errorf("can't load configuration: %w", err)
+	}
+
+	baseURL := stack.PackageRegistryBaseURL(profile, appConfig)
+	eprClient, err := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	if err != nil {
+		return fmt.Errorf("failed to create package registry client: %w", err)
+	}
+	requiredInputsResolver := requiredinputs.NewRequiredInputsResolver(eprClient)
+
 	withOpts := []rally.OptionFunc{
 		rally.WithVariant(variant),
 		rally.WithBenchmarkName(benchName),
@@ -344,6 +358,7 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 		rally.WithRallyPackageFromRegistry(packageName, packageVersion),
 		rally.WithRallyCorpusAtPath(corpusAtPath),
 		rally.WithRepositoryRoot(repositoryRoot),
+		rally.WithRequiredInputsResolver(requiredInputsResolver),
 	}
 
 	esMetricsClient, err := initializeESMetricsClient(ctx)
@@ -506,6 +521,18 @@ func streamCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't create Kibana client: %w", err)
 	}
 
+	appConfig, err := install.Configuration()
+	if err != nil {
+		return fmt.Errorf("can't load configuration: %w", err)
+	}
+
+	baseURL := stack.PackageRegistryBaseURL(profile, appConfig)
+	eprClient, err := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	if err != nil {
+		return fmt.Errorf("failed to create package registry client: %w", err)
+	}
+	requiredInputsResolver := requiredinputs.NewRequiredInputsResolver(eprClient)
+
 	withOpts := []stream.OptionFunc{
 		stream.WithVariant(variant),
 		stream.WithBenchmarkName(benchName),
@@ -519,6 +546,7 @@ func streamCommandAction(cmd *cobra.Command, args []string) error {
 		stream.WithKibanaClient(kc),
 		stream.WithProfile(profile),
 		stream.WithRepositoryRoot(repositoryRoot),
+		stream.WithRequiredInputsResolver(requiredInputsResolver),
 	}
 
 	runner := stream.NewStreamBenchmark(stream.NewOptions(withOpts...))

cmd/build.go

@@ -16,6 +16,10 @@ import (
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/elastic-package/internal/profile"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
+	"github.com/elastic/elastic-package/internal/stack"
 )
 
 const buildLongDescription = `Use this command to build a package.
@@ -26,6 +30,8 @@ Built packages are served up by the Elastic Package Registry running locally (se
 
 Built packages can also be published to the global package registry service.
 
+When the package declares required input packages ("requires.input" in manifest.yml), the build downloads those input packages from the configured package registry (see "package_registry.base_url" in ~/.elastic-package/config.yml). The build then incorporates their policy and data stream templates, merges variable definitions into the integration manifest, bundles data stream field definitions, and resolves package: references on inputs and streams to the effective input types expected by Fleet. For details on using a local or custom registry during development, see the [HOWTO guide](./docs/howto/local_package_registry.md).
+
 For details on how to enable dependency management, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/dependency_management.md).`
 
 func setupBuildCommand() *cobraext.Command {
@@ -84,15 +90,28 @@ func buildCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	prof, err := profile.LoadProfile(appConfig.CurrentProfile())
+	if err != nil {
+		return fmt.Errorf("could not load profile: %w", err)
+	}
+	baseURL := stack.PackageRegistryBaseURL(prof, appConfig)
+	eprClient, err := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, prof)...)
+	if err != nil {
+		return fmt.Errorf("failed to create package registry client: %w", err)
+	}
+
+	requiredInputsResolver := requiredinputs.NewRequiredInputsResolver(eprClient)
+
 	target, err := builder.BuildPackage(builder.BuildOptions{
-		PackageRoot:    packageRoot,
-		BuildDir:       buildDir,
-		CreateZip:      createZip,
-		SignPackage:    signPackage,
-		SkipValidation: skipValidation,
-		RepositoryRoot: repositoryRoot,
-		UpdateReadmes:  true,
-		SchemaURLs:     appConfig.SchemaURLs(),
+		PackageRoot:            packageRoot,
+		BuildDir:               buildDir,
+		CreateZip:              createZip,
+		SignPackage:            signPackage,
+		SkipValidation:         skipValidation,
+		RepositoryRoot:         repositoryRoot,
+		UpdateReadmes:          true,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 	if err != nil {
 		return fmt.Errorf("building package failed: %w", err)

cmd/install.go

@@ -16,6 +16,8 @@ import (
 	"github.com/elastic/elastic-package/internal/kibana"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/packages/installer"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
 	"github.com/elastic/elastic-package/internal/stack"
 )
 
@@ -96,13 +98,22 @@ func installCommandAction(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	baseURL := stack.PackageRegistryBaseURL(profile, appConfig)
+	eprClient, err := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	if err != nil {
+		return fmt.Errorf("failed to create package registry client: %w", err)
+	}
+
+	requiredInputsResolver := requiredinputs.NewRequiredInputsResolver(eprClient)
+
 	installer, err := installer.NewForPackage(installer.Options{
-		Kibana:         kibanaClient,
-		PackageRoot:    packageRoot,
-		SkipValidation: skipValidation,
-		ZipPath:        zipPathFile,
-		RepositoryRoot: repositoryRoot,
-		SchemaURLs:     appConfig.SchemaURLs(),
+		Kibana:                 kibanaClient,
+		PackageRoot:            packageRoot,
+		SkipValidation:         skipValidation,
+		ZipPath:                zipPathFile,
+		RepositoryRoot:         repositoryRoot,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 	if err != nil {
 		return fmt.Errorf("package installation failed: %w", err)

cmd/status.go

@@ -124,7 +124,10 @@ func statusCommandAction(cmd *cobra.Command, args []string) error {
 
 	// Create registry client with configured URL
 	// Currently, this command does not use profile, so we take the URL from the application configuration
-	registryClient := registry.NewClient(appConfig.PackageRegistryBaseURL())
+	registryClient, err := registry.NewClient(appConfig.PackageRegistryBaseURL())
+	if err != nil {
+		return fmt.Errorf("failed to create package registry client: %w", err)
+	}
 
 	options := registry.SearchOptions{
 		All:           showAll,

cmd/testrunner.go

@@ -21,6 +21,8 @@ import (
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
 	"github.com/elastic/elastic-package/internal/signal"
 	"github.com/elastic/elastic-package/internal/stack"
 	"github.com/elastic/elastic-package/internal/testrunner"
@@ -760,6 +762,12 @@ func testRunnerScriptCommandAction(cmd *cobra.Command, args []string) error {
 
 	opts.Package = manifest.Name
 
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return err
+	}
+	opts.Profile = profile
+
 	var results []testrunner.TestResult
 	err = script.Run(&results, cmd.OutOrStderr(), opts)
 	if err != nil {
@@ -870,19 +878,27 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	baseURL := stack.PackageRegistryBaseURL(profile, appConfig)
+	eprClient, err := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	if err != nil {
+		return fmt.Errorf("failed to create package registry client: %w", err)
+	}
+	requiredInputsResolver := requiredinputs.NewRequiredInputsResolver(eprClient)
+
 	logger.Info(version.Version())
 	logger.Infof("elastic-stack: %s", stackVersion.Version())
 	runner := policy.NewPolicyTestRunner(policy.PolicyTestRunnerOptions{
-		PackageRoot:        packageRoot,
-		KibanaClient:       kibanaClient,
-		DataStreams:        dataStreams,
-		FailOnMissingTests: failOnMissing,
-		GenerateTestResult: generateTestResult,
-		GlobalTestConfig:   globalTestConfig.Policy,
-		WithCoverage:       testCoverage,
-		CoverageType:       testCoverageFormat,
-		RepositoryRoot:     repositoryRoot,
-		SchemaURLs:         appConfig.SchemaURLs(),
+		PackageRoot:            packageRoot,
+		KibanaClient:           kibanaClient,
+		DataStreams:            dataStreams,
+		FailOnMissingTests:     failOnMissing,
+		GenerateTestResult:     generateTestResult,
+		GlobalTestConfig:       globalTestConfig.Policy,
+		WithCoverage:           testCoverage,
+		CoverageType:           testCoverageFormat,
+		RepositoryRoot:         repositoryRoot,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 
 	results, err := testrunner.RunSuite(ctx, runner)

docs/howto/dependency_management.md

@@ -11,10 +11,22 @@ which field definition was correct, maintenance and typo correction process was
 The described situation brought us to a point in time when a simple dependency management was a requirement to maintain
 all used fields, especially ones imported from external sources.
 
+Elastic Packages support two kinds of build-time dependency:
+
+- **Field dependencies** — import field definitions from external schemas (e.g. ECS) using
+  `_dev/build/build.yml`. Resolved from Git references and cached locally.
+- **Package dependencies** — composable (integration) packages can depend on input and content packages
+  declared under `requires` in `manifest.yml`. **Input package** dependencies are resolved
+  at build time by downloading from the package registry. **Content package** dependencies are
+  resolved at runtime by Fleet.
+
+Both are described in the sections below.
+
 ## Principles of operation
 
-Currently Elastic Packages support build-time dependencies that can be used as external field sources. They use a flat
-dependency model represented with an additional build manifest, stored in an optional YAML file - `_dev/build/build.yml`:
+Currently Elastic Packages support build-time field dependencies that can be used as external
+field sources. They use a flat dependency model represented with an additional build manifest,
+stored in an optional YAML file - `_dev/build/build.yml`:
 
 ```yaml
 dependencies:
@@ -83,4 +95,51 @@ and use a following field definition:
 ```yaml
 - name: event.category
   external: ecs
-```
+```
+
+## Composable packages and the package registry
+
+Composable (integration) packages can also depend on input or content packages by declaring them under
+`requires` in `manifest.yml`. Depending on the package type, dependencies are resolved
+differently: **input package** dependencies are fetched at build time; **content package**
+dependencies are resolved at runtime by Fleet.
+
+```yaml
+requires:
+  input:
+    - package: sql_input
+      version: "0.2.0"
+```
+
+This type of dependency is resolved at **build time** by downloading the required input package
+from the **package registry**. During `elastic-package build`, elastic-package fetches those
+packages and updates the built integration: it bundles agent templates (policy and data stream),
+merges variable definitions from the input packages into the composable manifest, adds data
+stream field definitions where configured, and rewrites `package:` references on inputs and
+streams to the concrete input types Fleet needs. Fleet still merges policy-specific values at
+policy creation time.
+
+Unlike field-level dependencies (which are resolved from Git references and cached locally),
+package dependencies are fetched from the configured package registry URL
+(`package_registry.base_url` in `~/.elastic-package/config.yml`, defaulting to
+`https://epr.elastic.co`).
+
+For details on using a local or custom registry when the required input packages are still
+under development, see [HOWTO: Use a local or custom package registry](./local_package_registry.md).
+
+### Linked files (`*.link`) and `template_path`
+
+Some repositories share agent templates using **link files** (files ending in `.link` that
+point at shared content). During `elastic-package build`, linked content is copied into the
+build output under the **target** path (the link filename without the `.link` suffix).
+
+Composable bundling (`requires.input`) runs **after** linked files are materialized in the
+build directory. In `manifest.yml`, always set `template_path` / `template_paths` to those
+**materialized** names (for example `owned.hbs`), **not** the stub name (`owned.hbs.link`).
+Fleet and the builder resolve templates by the names declared in the manifest; the `.link`
+file exists only in the source tree.
+
+A small manual fixture that combines `requires.input` with a linked policy input template
+lives under `test/manual_packages/required_inputs/with_linked_template_path/`. Automated
+coverage is in `TestBundleInputPackageTemplates_PreservesLinkedTemplateTargetPath` in
+`internal/requiredinputs/requiredinputs_test.go`.