bundle templates of required input packages

README.md

@@ -238,6 +238,8 @@ Built packages are served up by the Elastic Package Registry running locally (se
 
 Built packages can also be published to the global package registry service.
 
+When the package declares required input packages ("requires.input" in manifest.yml), the build downloads those input packages from the configured package registry (see "package_registry.base_url" in ~/.elastic-package/config.yml). For details on using a local or custom registry during development, see the [HOWTO guide](./docs/howto/local_package_registry.md).
+
 For details on how to enable dependency management, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/dependency_management.md).
 
 ### `elastic-package changelog`

cmd/benchmark.go

@@ -29,6 +29,8 @@ import (
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
 	"github.com/elastic/elastic-package/internal/signal"
 	"github.com/elastic/elastic-package/internal/stack"
 	"github.com/elastic/elastic-package/internal/testrunner"
@@ -331,6 +333,18 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't create Kibana client: %w", err)
 	}
 
+	appConfig, err := install.Configuration()
+	if err != nil {
+		return fmt.Errorf("can't load configuration: %w", err)
+	}
+
+	baseURL := appConfig.PackageRegistryBaseURL()
+	eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient)
+	if err != nil {
+		return fmt.Errorf("creating required inputs resolver failed: %w", err)
+	}
+
 	withOpts := []rally.OptionFunc{
 		rally.WithVariant(variant),
 		rally.WithBenchmarkName(benchName),
@@ -344,6 +358,7 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 		rally.WithRallyPackageFromRegistry(packageName, packageVersion),
 		rally.WithRallyCorpusAtPath(corpusAtPath),
 		rally.WithRepositoryRoot(repositoryRoot),
+		rally.WithRequiredInputsResolver(requiredInputsResolver),
 	}
 
 	esMetricsClient, err := initializeESMetricsClient(ctx)
@@ -506,6 +521,18 @@ func streamCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't create Kibana client: %w", err)
 	}
 
+	appConfig, err := install.Configuration()
+	if err != nil {
+		return fmt.Errorf("can't load configuration: %w", err)
+	}
+
+	baseURL := appConfig.PackageRegistryBaseURL()
+	eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient)
+	if err != nil {
+		return fmt.Errorf("creating required inputs resolver failed: %w", err)
+	}
+
 	withOpts := []stream.OptionFunc{
 		stream.WithVariant(variant),
 		stream.WithBenchmarkName(benchName),
@@ -519,6 +546,7 @@ func streamCommandAction(cmd *cobra.Command, args []string) error {
 		stream.WithKibanaClient(kc),
 		stream.WithProfile(profile),
 		stream.WithRepositoryRoot(repositoryRoot),
+		stream.WithRequiredInputsResolver(requiredInputsResolver),
 	}
 
 	runner := stream.NewStreamBenchmark(stream.NewOptions(withOpts...))

cmd/build.go

@@ -16,6 +16,10 @@ import (
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/elastic-package/internal/profile"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
+	"github.com/elastic/elastic-package/internal/stack"
 )
 
 const buildLongDescription = `Use this command to build a package.
@@ -26,6 +30,8 @@ Built packages are served up by the Elastic Package Registry running locally (se
 
 Built packages can also be published to the global package registry service.
 
+When the package declares required input packages ("requires.input" in manifest.yml), the build downloads those input packages from the configured package registry (see "package_registry.base_url" in ~/.elastic-package/config.yml). For details on using a local or custom registry during development, see the [HOWTO guide](./docs/howto/local_package_registry.md).
+
 For details on how to enable dependency management, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/dependency_management.md).`
 
 func setupBuildCommand() *cobraext.Command {
@@ -84,15 +90,28 @@ func buildCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	baseURL := appConfig.PackageRegistryBaseURL()
+	prof, err := profile.LoadProfile(appConfig.CurrentProfile())
+	if err != nil {
+		return fmt.Errorf("could not load profile: %w", err)
+	}
+	eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, prof)...)
+
+	requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient)
+	if err != nil {
+		return fmt.Errorf("creating required inputs resolver failed: %w", err)
+	}
+
 	target, err := builder.BuildPackage(builder.BuildOptions{
-		PackageRoot:    packageRoot,
-		BuildDir:       buildDir,
-		CreateZip:      createZip,
-		SignPackage:    signPackage,
-		SkipValidation: skipValidation,
-		RepositoryRoot: repositoryRoot,
-		UpdateReadmes:  true,
-		SchemaURLs:     appConfig.SchemaURLs(),
+		PackageRoot:            packageRoot,
+		BuildDir:               buildDir,
+		CreateZip:              createZip,
+		SignPackage:            signPackage,
+		SkipValidation:         skipValidation,
+		RepositoryRoot:         repositoryRoot,
+		UpdateReadmes:          true,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 	if err != nil {
 		return fmt.Errorf("building package failed: %w", err)

cmd/install.go

@@ -15,6 +15,8 @@ import (
 	"github.com/elastic/elastic-package/internal/kibana"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/packages/installer"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
 	"github.com/elastic/elastic-package/internal/stack"
 )
 
@@ -92,13 +94,22 @@ func installCommandAction(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	baseURL := stack.PackageRegistryBaseURL(profile, appConfig)
+	eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+
+	requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient)
+	if err != nil {
+		return fmt.Errorf("creating required inputs resolver failed: %w", err)
+	}
+
 	installer, err := installer.NewForPackage(installer.Options{
-		Kibana:         kibanaClient,
-		PackageRoot:    packageRoot,
-		SkipValidation: skipValidation,
-		ZipPath:        zipPathFile,
-		RepositoryRoot: repositoryRoot,
-		SchemaURLs:     appConfig.SchemaURLs(),
+		Kibana:                 kibanaClient,
+		PackageRoot:            packageRoot,
+		SkipValidation:         skipValidation,
+		ZipPath:                zipPathFile,
+		RepositoryRoot:         repositoryRoot,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 	if err != nil {
 		return fmt.Errorf("package installation failed: %w", err)

cmd/testrunner.go

@@ -21,6 +21,8 @@ import (
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/elastic-package/internal/registry"
+	"github.com/elastic/elastic-package/internal/requiredinputs"
 	"github.com/elastic/elastic-package/internal/signal"
 	"github.com/elastic/elastic-package/internal/stack"
 	"github.com/elastic/elastic-package/internal/testrunner"
@@ -870,19 +872,27 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't load configuration: %w", err)
 	}
 
+	baseURL := appConfig.PackageRegistryBaseURL()
+	eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...)
+	requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient)
+	if err != nil {
+		return fmt.Errorf("creating required inputs resolver failed: %w", err)
+	}
+
 	logger.Info(version.Version())
 	logger.Infof("elastic-stack: %s", stackVersion.Version())
 	runner := policy.NewPolicyTestRunner(policy.PolicyTestRunnerOptions{
-		PackageRoot:        packageRoot,
-		KibanaClient:       kibanaClient,
-		DataStreams:        dataStreams,
-		FailOnMissingTests: failOnMissing,
-		GenerateTestResult: generateTestResult,
-		GlobalTestConfig:   globalTestConfig.Policy,
-		WithCoverage:       testCoverage,
-		CoverageType:       testCoverageFormat,
-		RepositoryRoot:     repositoryRoot,
-		SchemaURLs:         appConfig.SchemaURLs(),
+		PackageRoot:            packageRoot,
+		KibanaClient:           kibanaClient,
+		DataStreams:            dataStreams,
+		FailOnMissingTests:     failOnMissing,
+		GenerateTestResult:     generateTestResult,
+		GlobalTestConfig:       globalTestConfig.Policy,
+		WithCoverage:           testCoverage,
+		CoverageType:           testCoverageFormat,
+		RepositoryRoot:         repositoryRoot,
+		SchemaURLs:             appConfig.SchemaURLs(),
+		RequiredInputsResolver: requiredInputsResolver,
 	})
 
 	results, err := testrunner.RunSuite(ctx, runner)

docs/howto/dependency_management.md

@@ -11,10 +11,22 @@ which field definition was correct, maintenance and typo correction process was
 The described situation brought us to a point in time when a simple dependency management was a requirement to maintain
 all used fields, especially ones imported from external sources.
 
+Elastic Packages support two kinds of build-time dependency:
+
+- **Field dependencies** — import field definitions from external schemas (e.g. ECS) using
+  `_dev/build/build.yml`. Resolved from Git references and cached locally.
+- **Package dependencies** — composable (integration) packages can depend on input and content packages
+  declared under `requires` in `manifest.yml`. **Input package** dependencies are resolved
+  at build time by downloading from the package registry. **Content package** dependencies are
+  resolved at runtime by Fleet.
+
+Both are described in the sections below.
+
 ## Principles of operation
 
-Currently Elastic Packages support build-time dependencies that can be used as external field sources. They use a flat
-dependency model represented with an additional build manifest, stored in an optional YAML file - `_dev/build/build.yml`:
+Currently Elastic Packages support build-time field dependencies that can be used as external
+field sources. They use a flat dependency model represented with an additional build manifest,
+stored in an optional YAML file - `_dev/build/build.yml`:
 
 ```yaml
 dependencies:
@@ -83,4 +95,31 @@ and use a following field definition:
 ```yaml
 - name: event.category
   external: ecs
-```
+```
+
+## Composable packages and the package registry
+
+Composable (integration) packages can also depend on input or content packages by declaring them under
+`requires` in `manifest.yml`. Depending on the package type, dependencies are resolved
+differently: **input package** dependencies are fetched at build time; **content package**
+dependencies are resolved at runtime by Fleet.
+
+```yaml
+requires:
+  input:
+    - package: sql_input
+      version: "0.2.0"
+```
+
+This type of dependency is resolved at **build time** by downloading the required input package
+from the **package registry**. During `elastic-package build`, agent templates from the
+required input packages are fetched and bundled into the built integration so that Fleet can
+merge them at policy creation time.
+
+Unlike field-level dependencies (which are resolved from Git references and cached locally),
+package dependencies are fetched from the configured package registry URL
+(`package_registry.base_url` in `~/.elastic-package/config.yml`, defaulting to
+`https://epr.elastic.co`).
+
+For details on using a local or custom registry when the required input packages are still
+under development, see [HOWTO: Use a local or custom package registry](./local_package_registry.md).