bundle templates of required input packages

README.md

@@ -238,6 +238,8 @@ Built packages are served up by the Elastic Package Registry running locally (se
 
 Built packages can also be published to the global package registry service.
 
+When the package declares required input packages ("requires.input" in manifest.yml), the build downloads those input packages from the configured package registry (see "package_registry.base_url" in ~/.elastic-package/config.yml). For details on using a local or custom registry during development, see the [HOWTO guide](./docs/howto/local_package_registry.md).
+
 For details on how to enable dependency management, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/dependency_management.md).
 
 ### `elastic-package changelog`

cmd/benchmark.go

@@ -29,6 +29,8 @@ import (
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
 	"github.com/elastic/elastic-package/internal/signal"
 	"github.com/elastic/elastic-package/internal/stack"
 	"github.com/elastic/elastic-package/internal/testrunner"
@@ -331,6 +333,18 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't create Kibana client: %w", err)
 	}
 
+	appConfig, err := install.Configuration()
+	if err != nil {
+		return fmt.Errorf("can't load configuration: %w", err)
+	}
+
+	baseURL := appConfig.PackageRegistryBaseURL()
+	eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient)
+	if err != nil {
+		return fmt.Errorf("creating required inputs resolver failed: %w", err)
+	}
+
 	withOpts := []rally.OptionFunc{
 		rally.WithVariant(variant),
 		rally.WithBenchmarkName(benchName),
@@ -344,6 +358,7 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 		rally.WithRallyPackageFromRegistry(packageName, packageVersion),
 		rally.WithRallyCorpusAtPath(corpusAtPath),
 		rally.WithRepositoryRoot(repositoryRoot),
+		rally.WithRequiredInputsResolver(requiredInputsResolver),
 	}
 
 	esMetricsClient, err := initializeESMetricsClient(ctx)
@@ -506,6 +521,18 @@ func streamCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't create Kibana client: %w", err)
 	}
 
+	appConfig, err := install.Configuration()
+	if err != nil {
+		return fmt.Errorf("can't load configuration: %w", err)
+	}
+
+	baseURL := appConfig.PackageRegistryBaseURL()
+	eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient)
+	if err != nil {
+		return fmt.Errorf("creating required inputs resolver failed: %w", err)
+	}
+
 	withOpts := []stream.OptionFunc{
 		stream.WithVariant(variant),
 		stream.WithBenchmarkName(benchName),
@@ -519,6 +546,7 @@ func streamCommandAction(cmd *cobra.Command, args []string) error {
 		stream.WithKibanaClient(kc),
 		stream.WithProfile(profile),
 		stream.WithRepositoryRoot(repositoryRoot),
+		stream.WithRequiredInputsResolver(requiredInputsResolver),
 	}
 
 	runner := stream.NewStreamBenchmark(stream.NewOptions(withOpts...))

cmd/build.go

@@ -16,6 +16,10 @@ import (
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/elastic-package/internal/profile"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
+	"github.com/elastic/elastic-package/internal/stack"
 )
 
 const buildLongDescription = `Use this command to build a package.
@@ -26,6 +30,8 @@ Built packages are served up by the Elastic Package Registry running locally (se
 
 Built packages can also be published to the global package registry service.
 
+When the package declares required input packages ("requires.input" in manifest.yml), the build downloads those input packages from the configured package registry (see "package_registry.base_url" in ~/.elastic-package/config.yml). For details on using a local or custom registry during development, see the [HOWTO guide](./docs/howto/local_package_registry.md).
+
 For details on how to enable dependency management, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/dependency_management.md).`
 
 func setupBuildCommand() *cobraext.Command {
@@ -84,15 +90,28 @@ func buildCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	baseURL := appConfig.PackageRegistryBaseURL()
+	prof, err := profile.LoadProfile(appConfig.CurrentProfile())
+	if err != nil {
+		return fmt.Errorf("could not load profile: %w", err)
+	}
+	eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, prof)...)
+
+	requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient)
+	if err != nil {
+		return fmt.Errorf("creating required inputs resolver failed: %w", err)
+	}
+
 	target, err := builder.BuildPackage(builder.BuildOptions{
-		PackageRoot:    packageRoot,
-		BuildDir:       buildDir,
-		CreateZip:      createZip,
-		SignPackage:    signPackage,
-		SkipValidation: skipValidation,
-		RepositoryRoot: repositoryRoot,
-		UpdateReadmes:  true,
-		SchemaURLs:     appConfig.SchemaURLs(),
+		PackageRoot:            packageRoot,
+		BuildDir:               buildDir,
+		CreateZip:              createZip,
+		SignPackage:            signPackage,
+		SkipValidation:         skipValidation,
+		RepositoryRoot:         repositoryRoot,
+		UpdateReadmes:          true,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 	if err != nil {
 		return fmt.Errorf("building package failed: %w", err)

cmd/install.go

@@ -15,6 +15,8 @@ import (
 	"github.com/elastic/elastic-package/internal/kibana"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/packages/installer"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
 	"github.com/elastic/elastic-package/internal/stack"
 )
 
@@ -92,13 +94,22 @@ func installCommandAction(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	baseURL := stack.PackageRegistryBaseURL(profile, appConfig)
+	eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+
+	requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient)
+	if err != nil {
+		return fmt.Errorf("creating required inputs resolver failed: %w", err)
+	}
+
 	installer, err := installer.NewForPackage(installer.Options{
-		Kibana:         kibanaClient,
-		PackageRoot:    packageRoot,
-		SkipValidation: skipValidation,
-		ZipPath:        zipPathFile,
-		RepositoryRoot: repositoryRoot,
-		SchemaURLs:     appConfig.SchemaURLs(),
+		Kibana:                 kibanaClient,
+		PackageRoot:            packageRoot,
+		SkipValidation:         skipValidation,
+		ZipPath:                zipPathFile,
+		RepositoryRoot:         repositoryRoot,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 	if err != nil {
 		return fmt.Errorf("package installation failed: %w", err)

cmd/testrunner.go

@@ -21,6 +21,8 @@ import (
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
 	"github.com/elastic/elastic-package/internal/signal"
 	"github.com/elastic/elastic-package/internal/stack"
 	"github.com/elastic/elastic-package/internal/testrunner"
@@ -870,19 +872,27 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	baseURL := appConfig.PackageRegistryBaseURL()
+	eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient)
+	if err != nil {
+		return fmt.Errorf("creating required inputs resolver failed: %w", err)
+	}
+
 	logger.Info(version.Version())
 	logger.Infof("elastic-stack: %s", stackVersion.Version())
 	runner := policy.NewPolicyTestRunner(policy.PolicyTestRunnerOptions{
-		PackageRoot:        packageRoot,
-		KibanaClient:       kibanaClient,
-		DataStreams:        dataStreams,
-		FailOnMissingTests: failOnMissing,
-		GenerateTestResult: generateTestResult,
-		GlobalTestConfig:   globalTestConfig.Policy,
-		WithCoverage:       testCoverage,
-		CoverageType:       testCoverageFormat,
-		RepositoryRoot:     repositoryRoot,
-		SchemaURLs:         appConfig.SchemaURLs(),
+		PackageRoot:            packageRoot,
+		KibanaClient:           kibanaClient,
+		DataStreams:            dataStreams,
+		FailOnMissingTests:     failOnMissing,
+		GenerateTestResult:     generateTestResult,
+		GlobalTestConfig:       globalTestConfig.Policy,
+		WithCoverage:           testCoverage,
+		CoverageType:           testCoverageFormat,
+		RepositoryRoot:         repositoryRoot,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 
 	results, err := testrunner.RunSuite(ctx, runner)

docs/howto/dependency_management.md

@@ -11,10 +11,22 @@ which field definition was correct, maintenance and typo correction process was
 The described situation brought us to a point in time when a simple dependency management was a requirement to maintain
 all used fields, especially ones imported from external sources.
 
+Elastic Packages support two kinds of build-time dependency:
+
+- **Field dependencies** — import field definitions from external schemas (e.g. ECS) using
+  `_dev/build/build.yml`. Resolved from Git references and cached locally.
+- **Package dependencies** — composable (integration) packages can depend on input and content packages
+  declared under `requires` in `manifest.yml`. **Input package** dependencies are resolved
+  at build time by downloading from the package registry. **Content package** dependencies are
+  resolved at runtime by Fleet.
+
+Both are described in the sections below.
+
 ## Principles of operation
 
-Currently Elastic Packages support build-time dependencies that can be used as external field sources. They use a flat
-dependency model represented with an additional build manifest, stored in an optional YAML file - `_dev/build/build.yml`:
+Currently Elastic Packages support build-time field dependencies that can be used as external
+field sources. They use a flat dependency model represented with an additional build manifest,
+stored in an optional YAML file - `_dev/build/build.yml`:
 
 ```yaml
 dependencies:
@@ -83,4 +95,31 @@ and use a following field definition:
 ```yaml
 - name: event.category
   external: ecs
-```
+```
+
+## Composable packages and the package registry
+
+Composable (integration) packages can also depend on input or content packages by declaring them under
+`requires` in `manifest.yml`. Depending on the package type, dependencies are resolved
+differently: **input package** dependencies are fetched at build time; **content package**
+dependencies are resolved at runtime by Fleet.
+
+```yaml
+requires:
+  input:
+    - package: sql_input
+      version: "0.2.0"
+```
+
+This type of dependency is resolved at **build time** by downloading the required input package
+from the **package registry**. During `elastic-package build`, agent templates from the
+required input packages are fetched and bundled into the built integration so that Fleet can
+merge them at policy creation time.
+
+Unlike field-level dependencies (which are resolved from Git references and cached locally),
+package dependencies are fetched from the configured package registry URL
+(`package_registry.base_url` in `~/.elastic-package/config.yml`, defaulting to
+`https://epr.elastic.co`).
+
+For details on using a local or custom registry when the required input packages are still
+under development, see [HOWTO: Use a local or custom package registry](./local_package_registry.md).

docs/howto/local_package_registry.md

@@ -0,0 +1,114 @@
+# HOWTO: Use a local or custom package registry for composable integrations
+
+## Overview
+
+Composable (integration) packages can declare required input packages in their `manifest.yml`
+under `requires.input`. When you run `elastic-package build` or `elastic-package install`,
+elastic-package resolves those dependencies by downloading them from the **package registry**.
+By default it uses the production registry at `https://epr.elastic.co`.
+
+This guide explains how to point elastic-package at a local or custom registry, which is
+useful when the required input packages are still under development and not yet published to
+the production registry.
+
+For field-level build-time dependencies (ECS, `_dev/build/build.yml`), see
+[HOWTO: Enable dependency management](./dependency_management.md).
+
+## Prerequisites
+
+- An integration package that declares `requires.input` in its `manifest.yml`, for example:
+
+```yaml
+requires:
+  input:
+    - package: sql_input
+      version: "0.2.0"
+```
+
+- Optionally, a running local package registry that serves the required input packages.
+
+## Option 1: Use the built-in stack registry (recommended)
+
+`elastic-package stack up` (with the default compose provider) automatically starts a local
+package registry that serves all packages found under the `build/packages/` directory. You can
+use this registry to serve locally-built input packages without running any additional
+infrastructure.
+
+```shell
+# 1. Build the required input package — this places the built package under build/packages/
+cd /path/to/sql_input
+elastic-package build
+
+# 2. Start the Elastic Stack from anywhere inside the repository.
+#    elastic-package resolves the repository root automatically and the bundled
+#    registry picks up build/packages/ from there.
+elastic-package stack up -v -d
+```
+
+Then configure `~/.elastic-package/config.yml` to use the stack's local registry for
+`elastic-package build` and `elastic-package install`:
+
+```yaml
+package_registry:
+  base_url: http://localhost:8080
+```
+
+This setting is read by `elastic-package build`, `elastic-package install`, and
+`elastic-package status` when they need to contact the registry. It defaults to
+`https://epr.elastic.co` when not set.
+
+> **Note:** This setting does not change the package registry container that the Elastic Stack
+> itself uses (served by `elastic-package stack`). To also redirect the stack, see
+> [Option 2](#option-2-configure-the-registry-url-per-profile) below.
+
+### Alternative: standalone package registry container
+
+If you are not running `elastic-package stack`, you can start a standalone registry container.
+Use a port other than `8080` to avoid conflicting with the stack's built-in registry:
+
+```shell
+# Build your input package first
+cd /path/to/sql_input
+elastic-package build
+
+# Start a standalone registry on port 8081
+docker run --rm -p 8081:8080 \
+  -v /path/to/build/packages:/packages/package \
+  docker.elastic.co/package-registry/distribution:latest \
+  /package-registry serve --packages-path /packages
+```
+
+Then point `package_registry.base_url` at `http://localhost:8081` and run
+`elastic-package build` from your integration package directory.
+
+## Option 2: Configure the registry URL per profile
+
+If you run `elastic-package stack` and want both the **build tools** and the **stack's Fleet**
+to use the same custom registry, configure the active profile
+(e.g. `~/.elastic-package/profiles/default/config.yml`):
+
+```yaml
+# URL for the stack's package registry container to proxy requests to
+stack.epr.proxy_to: http://host.docker.internal:8080
+
+# URL for the stack's package registry base endpoint
+stack.epr.base_url: http://localhost:8080
+```
+
+Profile settings take precedence over the global `package_registry.base_url` for
+stack-related behavior. The priority order is:
+
+1. `stack.epr.proxy_to` (profile) — used as the proxy target for the local registry container
+2. `stack.epr.base_url` (profile) — used as the registry base URL
+3. `package_registry.base_url` (global `~/.elastic-package/config.yml`)
+4. `https://epr.elastic.co` (production fallback)
+
+For more details on profiles, see the
+[Elastic Package profiles section of the README](../../README.md#elastic-package-profiles).
+
+## Summary
+
+| Goal | Configuration |
+|------|--------------|
+| Override registry for `build` / `install` | `package_registry.base_url` in `~/.elastic-package/config.yml` |
+| Override registry for `stack` (Fleet) | `stack.epr.base_url` / `stack.epr.proxy_to` in the active profile `config.yml` |