diff --git a/README.md b/README.md index fb7e44fd17..8f7e2ba03d 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,8 @@ Built packages are served up by the Elastic Package Registry running locally (se Built packages can also be published to the global package registry service. +When the package declares required input packages ("requires.input" in manifest.yml), the build downloads those input packages from the configured package registry (see "package_registry.base_url" in ~/.elastic-package/config.yml). For details on using a local or custom registry during development, see the [HOWTO guide](./docs/howto/local_package_registry.md). + For details on how to enable dependency management, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/dependency_management.md). ### `elastic-package changelog` diff --git a/cmd/benchmark.go b/cmd/benchmark.go index 45cde94d25..8e61ae9a65 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -29,6 +29,8 @@ import ( "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/registry" + "github.com/elastic/elastic-package/internal/requiredinputs" "github.com/elastic/elastic-package/internal/signal" "github.com/elastic/elastic-package/internal/stack" "github.com/elastic/elastic-package/internal/testrunner" @@ -331,6 +333,18 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't create Kibana client: %w", err) } + appConfig, err := install.Configuration() + if err != nil { + return fmt.Errorf("can't load configuration: %w", err) + } + + baseURL := appConfig.PackageRegistryBaseURL() + eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...) + requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient) + if err != nil { + return fmt.Errorf("creating required inputs resolver failed: %w", err) + } + withOpts := []rally.OptionFunc{ rally.WithVariant(variant), rally.WithBenchmarkName(benchName), @@ -344,6 +358,7 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error { rally.WithRallyPackageFromRegistry(packageName, packageVersion), rally.WithRallyCorpusAtPath(corpusAtPath), rally.WithRepositoryRoot(repositoryRoot), + rally.WithRequiredInputsResolver(requiredInputsResolver), } esMetricsClient, err := initializeESMetricsClient(ctx) @@ -506,6 +521,18 @@ func streamCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't create Kibana client: %w", err) } + appConfig, err := install.Configuration() + if err != nil { + return fmt.Errorf("can't load configuration: %w", err) + } + + baseURL := appConfig.PackageRegistryBaseURL() + eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...) + requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient) + if err != nil { + return fmt.Errorf("creating required inputs resolver failed: %w", err) + } + withOpts := []stream.OptionFunc{ stream.WithVariant(variant), stream.WithBenchmarkName(benchName), @@ -519,6 +546,7 @@ func streamCommandAction(cmd *cobra.Command, args []string) error { stream.WithKibanaClient(kc), stream.WithProfile(profile), stream.WithRepositoryRoot(repositoryRoot), + stream.WithRequiredInputsResolver(requiredInputsResolver), } runner := stream.NewStreamBenchmark(stream.NewOptions(withOpts...)) diff --git a/cmd/build.go b/cmd/build.go index b101a70133..86c1c0b308 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -16,6 +16,10 @@ import ( "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/registry" + "github.com/elastic/elastic-package/internal/requiredinputs" + "github.com/elastic/elastic-package/internal/stack" ) const buildLongDescription = `Use this command to build a package. @@ -26,6 +30,8 @@ Built packages are served up by the Elastic Package Registry running locally (se Built packages can also be published to the global package registry service. +When the package declares required input packages ("requires.input" in manifest.yml), the build downloads those input packages from the configured package registry (see "package_registry.base_url" in ~/.elastic-package/config.yml). For details on using a local or custom registry during development, see the [HOWTO guide](./docs/howto/local_package_registry.md). + For details on how to enable dependency management, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/dependency_management.md).` func setupBuildCommand() *cobraext.Command { @@ -84,15 +90,28 @@ func buildCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't load configuration: %w", err) } + baseURL := appConfig.PackageRegistryBaseURL() + prof, err := profile.LoadProfile(appConfig.CurrentProfile()) + if err != nil { + return fmt.Errorf("could not load profile: %w", err) + } + eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, prof)...) + + requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient) + if err != nil { + return fmt.Errorf("creating required inputs resolver failed: %w", err) + } + target, err := builder.BuildPackage(builder.BuildOptions{ - PackageRoot: packageRoot, - BuildDir: buildDir, - CreateZip: createZip, - SignPackage: signPackage, - SkipValidation: skipValidation, - RepositoryRoot: repositoryRoot, - UpdateReadmes: true, - SchemaURLs: appConfig.SchemaURLs(), + PackageRoot: packageRoot, + BuildDir: buildDir, + CreateZip: createZip, + SignPackage: signPackage, + SkipValidation: skipValidation, + RepositoryRoot: repositoryRoot, + UpdateReadmes: true, + SchemaURLs: appConfig.SchemaURLs(), + RequiredInputsResolver: requiredInputsResolver, }) if err != nil { return fmt.Errorf("building package failed: %w", err) diff --git a/cmd/install.go b/cmd/install.go index 246df8d253..57c73f3b00 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -15,6 +15,8 @@ import ( "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/packages/installer" + "github.com/elastic/elastic-package/internal/registry" + "github.com/elastic/elastic-package/internal/requiredinputs" "github.com/elastic/elastic-package/internal/stack" ) @@ -92,13 +94,22 @@ func installCommandAction(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't load configuration: %w", err) } + baseURL := stack.PackageRegistryBaseURL(profile, appConfig) + eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...) + + requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient) + if err != nil { + return fmt.Errorf("creating required inputs resolver failed: %w", err) + } + installer, err := installer.NewForPackage(installer.Options{ - Kibana: kibanaClient, - PackageRoot: packageRoot, - SkipValidation: skipValidation, - ZipPath: zipPathFile, - RepositoryRoot: repositoryRoot, - SchemaURLs: appConfig.SchemaURLs(), + Kibana: kibanaClient, + PackageRoot: packageRoot, + SkipValidation: skipValidation, + ZipPath: zipPathFile, + RepositoryRoot: repositoryRoot, + SchemaURLs: appConfig.SchemaURLs(), + RequiredInputsResolver: requiredInputsResolver, }) if err != nil { return fmt.Errorf("package installation failed: %w", err) diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 655670f263..f985e0f68e 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -21,6 +21,8 @@ import ( "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/registry" + "github.com/elastic/elastic-package/internal/requiredinputs" "github.com/elastic/elastic-package/internal/signal" "github.com/elastic/elastic-package/internal/stack" "github.com/elastic/elastic-package/internal/testrunner" @@ -870,19 +872,27 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't load configuration: %w", err) } + baseURL := appConfig.PackageRegistryBaseURL() + eprClient := registry.NewClient(baseURL, stack.RegistryClientOptions(baseURL, profile)...) + requiredInputsResolver, err := requiredinputs.NewRequiredInputsResolver(eprClient) + if err != nil { + return fmt.Errorf("creating required inputs resolver failed: %w", err) + } + logger.Info(version.Version()) logger.Infof("elastic-stack: %s", stackVersion.Version()) runner := policy.NewPolicyTestRunner(policy.PolicyTestRunnerOptions{ - PackageRoot: packageRoot, - KibanaClient: kibanaClient, - DataStreams: dataStreams, - FailOnMissingTests: failOnMissing, - GenerateTestResult: generateTestResult, - GlobalTestConfig: globalTestConfig.Policy, - WithCoverage: testCoverage, - CoverageType: testCoverageFormat, - RepositoryRoot: repositoryRoot, - SchemaURLs: appConfig.SchemaURLs(), + PackageRoot: packageRoot, + KibanaClient: kibanaClient, + DataStreams: dataStreams, + FailOnMissingTests: failOnMissing, + GenerateTestResult: generateTestResult, + GlobalTestConfig: globalTestConfig.Policy, + WithCoverage: testCoverage, + CoverageType: testCoverageFormat, + RepositoryRoot: repositoryRoot, + SchemaURLs: appConfig.SchemaURLs(), + RequiredInputsResolver: requiredInputsResolver, }) results, err := testrunner.RunSuite(ctx, runner) diff --git a/docs/howto/dependency_management.md b/docs/howto/dependency_management.md index 16d96f97b7..b3c77276ef 100644 --- a/docs/howto/dependency_management.md +++ b/docs/howto/dependency_management.md @@ -11,10 +11,22 @@ which field definition was correct, maintenance and typo correction process was The described situation brought us to a point in time when a simple dependency management was a requirement to maintain all used fields, especially ones imported from external sources. +Elastic Packages support two kinds of build-time dependency: + +- **Field dependencies** — import field definitions from external schemas (e.g. ECS) using + `_dev/build/build.yml`. Resolved from Git references and cached locally. +- **Package dependencies** — composable (integration) packages can depend on input and content packages + declared under `requires` in `manifest.yml`. **Input package** dependencies are resolved + at build time by downloading from the package registry. **Content package** dependencies are + resolved at runtime by Fleet. + +Both are described in the sections below. + ## Principles of operation -Currently Elastic Packages support build-time dependencies that can be used as external field sources. They use a flat -dependency model represented with an additional build manifest, stored in an optional YAML file - `_dev/build/build.yml`: +Currently Elastic Packages support build-time field dependencies that can be used as external +field sources. They use a flat dependency model represented with an additional build manifest, +stored in an optional YAML file - `_dev/build/build.yml`: ```yaml dependencies: @@ -83,4 +95,31 @@ and use a following field definition: ```yaml - name: event.category external: ecs -``` \ No newline at end of file +``` + +## Composable packages and the package registry + +Composable (integration) packages can also depend on input or content packages by declaring them under +`requires` in `manifest.yml`. Depending on the package type, dependencies are resolved +differently: **input package** dependencies are fetched at build time; **content package** +dependencies are resolved at runtime by Fleet. + +```yaml +requires: + input: + - package: sql_input + version: "0.2.0" +``` + +This type of dependency is resolved at **build time** by downloading the required input package +from the **package registry**. During `elastic-package build`, agent templates from the +required input packages are fetched and bundled into the built integration so that Fleet can +merge them at policy creation time. + +Unlike field-level dependencies (which are resolved from Git references and cached locally), +package dependencies are fetched from the configured package registry URL +(`package_registry.base_url` in `~/.elastic-package/config.yml`, defaulting to +`https://epr.elastic.co`). + +For details on using a local or custom registry when the required input packages are still +under development, see [HOWTO: Use a local or custom package registry](./local_package_registry.md). \ No newline at end of file diff --git a/docs/howto/local_package_registry.md b/docs/howto/local_package_registry.md new file mode 100644 index 0000000000..3eb9f76302 --- /dev/null +++ b/docs/howto/local_package_registry.md @@ -0,0 +1,158 @@ +# HOWTO: Use a local or custom package registry for composable integrations + +## Overview + +Composable (integration) packages can declare required input packages in their `manifest.yml` +under `requires.input`. When you run `elastic-package build` or `elastic-package install`, +elastic-package resolves those dependencies by downloading them from the **package registry**. +By default it uses the production registry at `https://epr.elastic.co`. + +This guide explains how to point elastic-package at a local or custom registry, which is +useful when the required input packages are still under development and not yet published to +the production registry. + +For field-level build-time dependencies (ECS, `_dev/build/build.yml`), see +[HOWTO: Enable dependency management](./dependency_management.md). + +## Prerequisites + +- An integration package that declares `requires.input` in its `manifest.yml`, for example: + +```yaml +requires: + input: + - package: sql_input + version: "0.2.0" +``` + +- Optionally, a running local package registry that serves the required input packages. + +## Option 1: Use the built-in stack registry (recommended) + +`elastic-package stack up` (with the default compose provider) automatically starts a local +package registry container. The container runs in **proxy mode**: it serves packages found in +the repository's `build/packages/` directory and proxies all other package requests to the +production registry at `https://epr.elastic.co` (or to a custom upstream if configured). + +`elastic-package` discovers `build/packages/` by walking up from the current working +directory to the repository root, so you can run `elastic-package stack up` from anywhere +inside the repository. + +```shell +# 1. Build the required input package — this places the built package under build/packages/ +# at the repository root. +cd /path/to/sql_input +elastic-package build + +# 2. Start the Elastic Stack from anywhere inside the repository. +# The bundled registry picks up build/packages/ from the repository root. +elastic-package stack up -v -d +``` + +Then configure `~/.elastic-package/config.yml` to use the stack's local registry for +`elastic-package build`, `elastic-package test`, `elastic-package benchmark`, and +`elastic-package status`: + +```yaml +package_registry: + base_url: http://localhost:8080 +``` + +This setting defaults to `https://epr.elastic.co` when not set. + +> **Note:** This setting does not change the package registry container that the Elastic Stack +> itself uses (served by `elastic-package stack`). To also redirect the stack's proxy target, +> see [Option 2](#option-2-configure-the-registry-url-per-profile) below. + +### Alternative: standalone package registry container + +If you are not running `elastic-package stack`, you can start a standalone registry container. +Use a port other than `8080` to avoid conflicting with the stack's built-in registry: + +```shell +# Build your input package first +cd /path/to/sql_input +elastic-package build + +# Start a standalone registry on port 8081, mounting the build/packages/ directory +# at the repository root (run from anywhere inside the repo, or adjust the path). +docker run --rm -p 8081:8080 \ + -v "$(git -C /path/to/repo rev-parse --show-toplevel)/build/packages":/packages/package-registry \ + docker.elastic.co/package-registry/package-registry:v1.37.0 +``` + +> **Note:** The mounted directory must contain at least one valid package (a `.zip` file or an +> extracted package directory). If the directory is empty, the registry exits immediately with +> `No local packages found.` +> +> **Note:** The registry image tag above matches `PackageRegistryBaseImage` in +> [`internal/stack/versions.go`](../../internal/stack/versions.go); that constant is what +> `elastic-package stack` uses and is updated by automation, while this document is not — +> check there when upgrading. + +Then point `package_registry.base_url` at `http://localhost:8081` and run +`elastic-package build` from your integration package directory. + +## Option 2: Configure the registry URL per profile + +Use this option when you want both the **build tools** and the **stack's Fleet** to use the +same custom or standalone registry — for example, a registry serving packages not yet +published to production. + +Assume your custom registry is running on the host at port `8082`. Configure the active +profile (e.g. `~/.elastic-package/profiles/default/config.yml`): + +```yaml +# The stack's package registry container will proxy non-local requests to this URL. +# Use host.docker.internal so the container can reach the host. +stack.epr.proxy_to: http://host.docker.internal:8082 + +# elastic-package install (and stack commands) will use this URL to contact the registry. +stack.epr.base_url: http://localhost:8082 +``` + +To also cover `elastic-package build`, `elastic-package test`, `elastic-package benchmark`, +and `elastic-package status` (which do not read profile settings), add the global setting: + +```yaml +# ~/.elastic-package/config.yml +package_registry: + base_url: http://localhost:8082 +``` + +### URL resolution reference + +**For `elastic-package build`, `test`, `benchmark`, `status`** (global config only): + +| Priority | Setting | +| -------- | ------- | +| 1 | `package_registry.base_url` in `~/.elastic-package/config.yml` | +| 2 | `https://epr.elastic.co` (production fallback) | + +**For `elastic-package install` and stack commands** (profile takes precedence): + +| Priority | Setting | +| -------- | ------- | +| 1 | `stack.epr.base_url` in the active profile `config.yml` | +| 2 | `package_registry.base_url` in `~/.elastic-package/config.yml` | +| 3 | `https://epr.elastic.co` (production fallback) | + +**For the stack registry's proxy target** (`EPR_PROXY_TO` inside the container): + +| Priority | Setting | +| -------- | ------- | +| 1 | `stack.epr.proxy_to` in the active profile `config.yml` | +| 2 | `stack.epr.base_url` in the active profile `config.yml` | +| 3 | `package_registry.base_url` in `~/.elastic-package/config.yml` | +| 4 | `https://epr.elastic.co` (production fallback) | + +For more details on profiles, see the +[Elastic Package profiles section of the README](../../README.md#elastic-package-profiles). + +## Summary + +| Goal | Configuration | +| ---- | ------------- | +| Override registry for `build` / `test` / `benchmark` / `status` | `package_registry.base_url` in `~/.elastic-package/config.yml` | +| Override registry for `install` and stack commands | `stack.epr.base_url` in the active profile `config.yml` | +| Override proxy target for the stack's registry container | `stack.epr.proxy_to` in the active profile `config.yml` | diff --git a/internal/benchrunner/runners/rally/options.go b/internal/benchrunner/runners/rally/options.go index 1696ec1bf2..be5a1c0145 100644 --- a/internal/benchrunner/runners/rally/options.go +++ b/internal/benchrunner/runners/rally/options.go @@ -11,26 +11,28 @@ import ( "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/requiredinputs" ) // Options contains benchmark runner options. type Options struct { - ESAPI *elasticsearch.API - KibanaClient *kibana.Client - DeferCleanup time.Duration - MetricsInterval time.Duration - ReindexData bool - ESMetricsAPI *elasticsearch.API - BenchName string - PackageRoot string - Variant string - Profile *profile.Profile - RallyTrackOutputDir string - DryRun bool - PackageName string - PackageVersion string - CorpusAtPath string - RepositoryRoot *os.Root + ESAPI *elasticsearch.API + KibanaClient *kibana.Client + DeferCleanup time.Duration + MetricsInterval time.Duration + ReindexData bool + ESMetricsAPI *elasticsearch.API + BenchName string + PackageRoot string + Variant string + Profile *profile.Profile + RallyTrackOutputDir string + DryRun bool + PackageName string + PackageVersion string + CorpusAtPath string + RepositoryRoot *os.Root + RequiredInputsResolver requiredinputs.Resolver } type ClientOptions struct { @@ -126,3 +128,9 @@ func WithRepositoryRoot(r *os.Root) OptionFunc { opts.RepositoryRoot = r } } + +func WithRequiredInputsResolver(r requiredinputs.Resolver) OptionFunc { + return func(opts *Options) { + opts.RequiredInputsResolver = r + } +} diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 66550389ae..17b55a8312 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -486,10 +486,11 @@ func (r *runner) installPackageFromRegistry(ctx context.Context, packageName, pa func (r *runner) installPackageFromPackageRoot(ctx context.Context) error { logger.Debug("Installing package...") installer, err := installer.NewForPackage(installer.Options{ - Kibana: r.options.KibanaClient, - PackageRoot: r.options.PackageRoot, - SkipValidation: true, - RepositoryRoot: r.options.RepositoryRoot, + Kibana: r.options.KibanaClient, + PackageRoot: r.options.PackageRoot, + SkipValidation: true, + RepositoryRoot: r.options.RepositoryRoot, + RequiredInputsResolver: r.options.RequiredInputsResolver, }) if err != nil { return fmt.Errorf("failed to initialize package installer: %w", err) diff --git a/internal/benchrunner/runners/stream/options.go b/internal/benchrunner/runners/stream/options.go index 7770f54b0a..32a7e7bcb1 100644 --- a/internal/benchrunner/runners/stream/options.go +++ b/internal/benchrunner/runners/stream/options.go @@ -11,22 +11,24 @@ import ( "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/requiredinputs" ) // Options contains benchmark runner options. type Options struct { - ESAPI *elasticsearch.API - KibanaClient *kibana.Client - BenchName string - BackFill time.Duration - EventsPerPeriod uint64 - PeriodDuration time.Duration - PerformCleanup bool - TimestampField string - PackageRoot string - Variant string - Profile *profile.Profile - RepositoryRoot *os.Root + ESAPI *elasticsearch.API + KibanaClient *kibana.Client + BenchName string + BackFill time.Duration + EventsPerPeriod uint64 + PeriodDuration time.Duration + PerformCleanup bool + TimestampField string + PackageRoot string + Variant string + Profile *profile.Profile + RepositoryRoot *os.Root + RequiredInputsResolver requiredinputs.Resolver } type ClientOptions struct { @@ -115,3 +117,9 @@ func WithRepositoryRoot(r *os.Root) OptionFunc { opts.RepositoryRoot = r } } + +func WithRequiredInputsResolver(r requiredinputs.Resolver) OptionFunc { + return func(opts *Options) { + opts.RequiredInputsResolver = r + } +} diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index a66277aa34..e02e92cba4 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -253,10 +253,11 @@ func (r *runner) installPackage(ctx context.Context) error { func (r *runner) installPackageFromPackageRoot(ctx context.Context) error { logger.Debug("Installing package...") installer, err := installer.NewForPackage(installer.Options{ - Kibana: r.options.KibanaClient, - PackageRoot: r.options.PackageRoot, - SkipValidation: true, - RepositoryRoot: r.options.RepositoryRoot, + Kibana: r.options.KibanaClient, + PackageRoot: r.options.PackageRoot, + SkipValidation: true, + RepositoryRoot: r.options.RepositoryRoot, + RequiredInputsResolver: r.options.RequiredInputsResolver, }) if err != nil { diff --git a/internal/builder/packages.go b/internal/builder/packages.go index 239fbf085e..bce6b9dbe9 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -18,6 +18,7 @@ import ( "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/requiredinputs" "github.com/elastic/elastic-package/internal/validation" ) @@ -31,11 +32,12 @@ type BuildOptions struct { BuildDir string // directory where all the built packages are placed and zipped packages are stored RepositoryRoot *os.Root - CreateZip bool - SignPackage bool - SkipValidation bool - UpdateReadmes bool - SchemaURLs fields.SchemaURLs + CreateZip bool + SignPackage bool + SkipValidation bool + UpdateReadmes bool + SchemaURLs fields.SchemaURLs + RequiredInputsResolver requiredinputs.Resolver } // BuildDirectory function locates the target build directory. If the directory doesn't exist, it will create it. @@ -232,6 +234,11 @@ func BuildPackage(options BuildOptions) (string, error) { return "", fmt.Errorf("resolving transform manifests failed: %w", err) } + err = options.RequiredInputsResolver.BundleInputPackageTemplates(buildPackageRoot) + if err != nil { + return "", fmt.Errorf("bundling input package templates failed: %w", err) + } + if options.UpdateReadmes { err = docs.UpdateReadmes(options.RepositoryRoot, options.PackageRoot, buildPackageRoot, options.SchemaURLs) if err != nil { diff --git a/internal/packages/archetype/package_test.go b/internal/packages/archetype/package_test.go index 965d9c7544..b7761e98f3 100644 --- a/internal/packages/archetype/package_test.go +++ b/internal/packages/archetype/package_test.go @@ -96,17 +96,26 @@ func createPackageDescriptorForTest(packageType, kibanaVersion string) PackageDe } } +type requiredInputsResolverMock struct { + BundleInputPackageTemplatesFunc func(buildPackageRoot string) error +} + +func (r *requiredInputsResolverMock) BundleInputPackageTemplates(buildPackageRoot string) error { + return nil +} + func buildPackage(t *testing.T, repositoryRoot *os.Root, packageRoot string) error { buildDir := filepath.Join(repositoryRoot.Name(), "build") err := os.MkdirAll(buildDir, 0o755) require.NoError(t, err) _, err = builder.BuildPackage(builder.BuildOptions{ - PackageRoot: packageRoot, - BuildDir: buildDir, - RepositoryRoot: repositoryRoot, - UpdateReadmes: true, - SchemaURLs: fields.SchemaURLs{}, + PackageRoot: packageRoot, + BuildDir: buildDir, + RepositoryRoot: repositoryRoot, + UpdateReadmes: true, + SchemaURLs: fields.SchemaURLs{}, + RequiredInputsResolver: &requiredInputsResolverMock{}, }) return err } diff --git a/internal/packages/installer/factory.go b/internal/packages/installer/factory.go index baa61375d1..2edd64b9c0 100644 --- a/internal/packages/installer/factory.go +++ b/internal/packages/installer/factory.go @@ -17,6 +17,7 @@ import ( "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/requiredinputs" "github.com/elastic/elastic-package/internal/validation" ) @@ -35,12 +36,13 @@ type Installer interface { // Options are the parameters used to build an installer. type Options struct { - Kibana *kibana.Client - PackageRoot string // Root path of the package to be installed. - ZipPath string - SkipValidation bool - RepositoryRoot *os.Root // Root of the repository where package source code is located. - SchemaURLs fields.SchemaURLs + Kibana *kibana.Client + PackageRoot string // Root path of the package to be installed. + ZipPath string + SkipValidation bool + RepositoryRoot *os.Root // Root of the repository where package source code is located. + SchemaURLs fields.SchemaURLs + RequiredInputsResolver requiredinputs.Resolver // Input dependency resolver for downloading input packages. } // NewForPackage creates a new installer for a package, given its root path, or its prebuilt zip. @@ -88,13 +90,14 @@ func NewForPackage(options Options) (Installer, error) { } target, err := builder.BuildPackage(builder.BuildOptions{ - PackageRoot: options.PackageRoot, - CreateZip: supportsUploadZip, - SignPackage: false, - SkipValidation: options.SkipValidation, - RepositoryRoot: options.RepositoryRoot, - UpdateReadmes: false, - SchemaURLs: options.SchemaURLs, + PackageRoot: options.PackageRoot, + CreateZip: supportsUploadZip, + SignPackage: false, + SkipValidation: options.SkipValidation, + RepositoryRoot: options.RepositoryRoot, + UpdateReadmes: false, + SchemaURLs: options.SchemaURLs, + RequiredInputsResolver: options.RequiredInputsResolver, }) if err != nil { return nil, fmt.Errorf("failed to build package: %v", err) diff --git a/internal/packages/packages.go b/internal/packages/packages.go index f86c57bed5..17b22fb7a2 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -179,8 +179,23 @@ type Variable struct { // Input is a single input configuration. type Input struct { - Type string `config:"type" json:"type" yaml:"type"` - Vars []Variable `config:"vars" json:"vars" yaml:"vars"` + Type string `config:"type" json:"type" yaml:"type"` + Package string `config:"package,omitempty" json:"package,omitempty" yaml:"package,omitempty"` + Vars []Variable `config:"vars" json:"vars" yaml:"vars"` + TemplatePath string `config:"template_path,omitempty" json:"template_path,omitempty" yaml:"template_path,omitempty"` + TemplatePaths []string `config:"template_paths,omitempty" json:"template_paths,omitempty" yaml:"template_paths,omitempty"` +} + +// PackageDependency describes a dependency on another package. +type PackageDependency struct { + Package string `config:"package" json:"package" yaml:"package"` + Version string `config:"version" json:"version" yaml:"version"` +} + +// Requires lists the packages that an integration package depends on. +type Requires struct { + Input []PackageDependency `config:"input,omitempty" json:"input,omitempty" yaml:"input,omitempty"` + Content []PackageDependency `config:"content,omitempty" json:"content,omitempty" yaml:"content,omitempty"` } // Source contains metadata about the source code of the package. @@ -221,10 +236,11 @@ type PolicyTemplate struct { Inputs []Input `config:"inputs,omitempty" json:"inputs,omitempty" yaml:"inputs,omitempty"` // For purposes of "input packages" - Input string `config:"input,omitempty" json:"input,omitempty" yaml:"input,omitempty"` - Type string `config:"type,omitempty" json:"type,omitempty" yaml:"type,omitempty"` - TemplatePath string `config:"template_path,omitempty" json:"template_path,omitempty" yaml:"template_path,omitempty"` - Vars []Variable `config:"vars,omitempty" json:"vars,omitempty" yaml:"vars,omitempty"` + Input string `config:"input,omitempty" json:"input,omitempty" yaml:"input,omitempty"` + Type string `config:"type,omitempty" json:"type,omitempty" yaml:"type,omitempty"` + TemplatePath string `config:"template_path,omitempty" json:"template_path,omitempty" yaml:"template_path,omitempty"` + TemplatePaths []string `config:"template_paths,omitempty" json:"template_paths,omitempty" yaml:"template_paths,omitempty"` + Vars []Variable `config:"vars,omitempty" json:"vars,omitempty" yaml:"vars,omitempty"` } // Owner defines package owners, either a single person or a team. @@ -257,6 +273,7 @@ type PackageManifest struct { Categories []string `config:"categories" json:"categories" yaml:"categories"` Agent Agent `config:"agent" json:"agent" yaml:"agent"` Elasticsearch *Elasticsearch `config:"elasticsearch" json:"elasticsearch" yaml:"elasticsearch"` + Requires *Requires `config:"requires,omitempty" json:"requires,omitempty" yaml:"requires,omitempty"` } type PackageDirNameAndManifest struct { @@ -319,11 +336,13 @@ type TransformDefinition struct { // Stream contains information about an input stream. type Stream struct { - Input string `config:"input" json:"input" yaml:"input"` - Title string `config:"title" json:"title" yaml:"title"` - Description string `config:"description" json:"description" yaml:"description"` - TemplatePath string `config:"template_path" json:"template_path" yaml:"template_path"` - Vars []Variable `config:"vars" json:"vars" yaml:"vars"` + Input string `config:"input" json:"input" yaml:"input"` + Package string `config:"package,omitempty" json:"package,omitempty" yaml:"package,omitempty"` + Title string `config:"title" json:"title" yaml:"title"` + Description string `config:"description" json:"description" yaml:"description"` + TemplatePath string `config:"template_path,omitempty" json:"template_path,omitempty" yaml:"template_path,omitempty"` + TemplatePaths []string `config:"template_paths,omitempty" json:"template_paths,omitempty" yaml:"template_paths,omitempty"` + Vars []Variable `config:"vars" json:"vars" yaml:"vars"` } // HasSource checks if a given index or data stream name maches the transform sources @@ -702,6 +721,20 @@ func ReadPackageManifestBytes(contents []byte) (*PackageManifest, error) { return &m, nil } +func ReadDataStreamManifestBytes(contents []byte) (*DataStreamManifest, error) { + cfg, err := yaml.NewConfig(contents, ucfg.PathSep(".")) + if err != nil { + return nil, fmt.Errorf("reading manifest file failed: %w", err) + } + + var m DataStreamManifest + err = cfg.Unpack(&m) + if err != nil { + return nil, fmt.Errorf("unpacking data stream manifest failed: %w", err) + } + return &m, nil +} + // ReadDataStreamManifest reads and parses the given data stream manifest file. func ReadDataStreamManifest(path string) (*DataStreamManifest, error) { cfg, err := yaml.NewConfigWithFile(path, ucfg.PathSep(".")) diff --git a/internal/registry/client.go b/internal/registry/client.go index 5f46cb585f..e4bec46f12 100644 --- a/internal/registry/client.go +++ b/internal/registry/client.go @@ -5,11 +5,13 @@ package registry import ( + "crypto/tls" "fmt" "io" "net/http" "net/url" + "github.com/elastic/elastic-package/internal/certs" "github.com/elastic/elastic-package/internal/logger" ) @@ -17,18 +19,59 @@ const ( ProductionURL = "https://epr.elastic.co" ) -// Client is responsible for exporting dashboards from Kibana. +// ClientOption is a functional option for the registry client. +type ClientOption func(*Client) + +// Client is responsible for communicating with the Package Registry API. type Client struct { - baseURL string + baseURL string + certificateAuthority string + tlsSkipVerify bool + httpClient *http.Client } // NewClient creates a new instance of the client. -func NewClient(baseURL string) *Client { - return &Client{ - baseURL: baseURL, +func NewClient(baseURL string, opts ...ClientOption) *Client { + c := &Client{baseURL: baseURL} + for _, opt := range opts { + opt(c) + } + c.httpClient, _ = c.newHTTPClient() + return c +} + +// CertificateAuthority sets the certificate authority to use for TLS verification. +func CertificateAuthority(path string) ClientOption { + return func(c *Client) { + c.certificateAuthority = path } } +// TLSSkipVerify disables TLS certificate verification (e.g. for local HTTPS registries). +func TLSSkipVerify() ClientOption { + return func(c *Client) { + c.tlsSkipVerify = true + } +} + +func (c *Client) newHTTPClient() (*http.Client, error) { + client := &http.Client{} + if c.tlsSkipVerify { + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } else if c.certificateAuthority != "" { + rootCAs, err := certs.SystemPoolWithCACertificate(c.certificateAuthority) + if err != nil { + return nil, fmt.Errorf("reading CA certificate: %w", err) + } + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: rootCAs}, + } + } + return client, nil +} + func (c *Client) get(resourcePath string) (int, []byte, error) { base, err := url.Parse(c.baseURL) if err != nil { @@ -50,7 +93,10 @@ func (c *Client) get(resourcePath string) (int, []byte, error) { return 0, nil, fmt.Errorf("could not create request to Package Registry API resource: %s: %w", resourcePath, err) } - client := http.Client{} + client := c.httpClient + if client == nil { + client = &http.Client{} + } resp, err := client.Do(req) if err != nil { return 0, nil, fmt.Errorf("could not send request to Package Registry API: %w", err) diff --git a/internal/registry/download.go b/internal/registry/download.go new file mode 100644 index 0000000000..c222002d33 --- /dev/null +++ b/internal/registry/download.go @@ -0,0 +1,57 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package registry + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + + "github.com/elastic/elastic-package/internal/logger" +) + +type packageMetadata struct { + Download string `json:"download"` +} + +// DownloadPackage fetches a package by name and version from the registry, +// saves the zip into destDir, and returns the path to the zip file. +func (c *Client) DownloadPackage(name, version, destDir string) (string, error) { + metadataPath := fmt.Sprintf("/package/%s/%s", name, version) + statusCode, body, err := c.get(metadataPath) + if err != nil { + return "", fmt.Errorf("fetching package metadata for %s-%s: %w", name, version, err) + } + if statusCode != http.StatusOK { + return "", fmt.Errorf("fetching package metadata for %s-%s: status %d: %s", name, version, statusCode, body) + } + + var meta packageMetadata + if err := json.Unmarshal(body, &meta); err != nil { + return "", fmt.Errorf("parsing package metadata for %s-%s: %w", name, version, err) + } + if meta.Download == "" { + return "", fmt.Errorf("package metadata for %s-%s has no download path", name, version) + } + + logger.Debugf("Downloading package %s-%s from %s", name, version, meta.Download) + statusCode, zipBytes, err := c.get(meta.Download) + if err != nil { + return "", fmt.Errorf("downloading package %s-%s: %w", name, version, err) + } + if statusCode != http.StatusOK { + return "", fmt.Errorf("downloading package %s-%s: status %d", name, version, statusCode) + } + + zipPath := filepath.Join(destDir, fmt.Sprintf("%s-%s.zip", name, version)) + if err := os.WriteFile(zipPath, zipBytes, 0644); err != nil { + return "", fmt.Errorf("writing package zip %s-%s: %w", name, version, err) + } + + logger.Debugf("Saved package %s-%s to %s", name, version, zipPath) + return zipPath, nil +} diff --git a/internal/requiredinputs/policytemplates.go b/internal/requiredinputs/policytemplates.go new file mode 100644 index 0000000000..8cb5645b8f --- /dev/null +++ b/internal/requiredinputs/policytemplates.go @@ -0,0 +1,194 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package requiredinputs + +import ( + "fmt" + "io/fs" + "os" + "path" + + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/packages" +) + +func (r *RequiredInputsResolver) bundlePolicyTemplatesInputPackageTemplates(manifestBytes []byte, manifest *packages.PackageManifest, inputPkgPaths map[string]string, buildRoot *os.Root) error { + + // parse the manifest YAML document preserving formatting for targeted modifications + // using manifestBytes allows us to preserve comments and formatting in the manifest when we update it with template paths from input packages + var doc yaml.Node + if err := yaml.Unmarshal(manifestBytes, &doc); err != nil { + return fmt.Errorf("failed to parse manifest YAML: %w", err) + } + + // for each policy template, with an input package reference: + // collect the templates from the input package and copy them to the agent/input directory of the build package + // then update the policy template manifest to include the copied templates as template_paths + for ptIdx, pt := range manifest.PolicyTemplates { + for inputIdx, input := range pt.Inputs { + if input.Package == "" { + continue + } + sourcePath, ok := inputPkgPaths[input.Package] + if !ok || sourcePath == "" { + return fmt.Errorf("failed to find input package %q referenced by policy template %q", input.Package, pt.Name) + } + inputPaths, err := r.collectAndCopyInputPkgPolicyTemplates(sourcePath, input.Package, buildRoot) + if err != nil { + return fmt.Errorf("failed to collect and copy input package policy templates: %w", err) + } + if len(inputPaths) == 0 { + continue + } + + // current manifest template paths + paths := make([]string, 0) + // if composable package has included custom template path or paths, include them + // if no template paths are included at the manifest, only the imported templates are included + if input.TemplatePath != "" { + paths = append(paths, input.TemplatePath) + } else if len(input.TemplatePaths) > 0 { + paths = append(paths, input.TemplatePaths...) + } + paths = append(inputPaths, paths...) + + if err := setInputPolicyTemplateTemplatePaths(&doc, ptIdx, inputIdx, paths); err != nil { + return fmt.Errorf("failed to update policy template manifest with input package templates: %w", err) + } + } + } + + // Serialise the updated YAML document back to disk. + updated, err := formatYAMLNode(&doc) + if err != nil { + return fmt.Errorf("failed to format updated manifest: %w", err) + } + if err := buildRoot.WriteFile("manifest.yml", updated, 0664); err != nil { + return fmt.Errorf("failed to write updated manifest: %w", err) + } + + return nil +} + +// collectAndCopyInputPkgPolicyTemplates collects the templates from the input package and copies them to the agent/input directory of the build package +// it returns the list of copied template names +func (r *RequiredInputsResolver) collectAndCopyInputPkgPolicyTemplates(inputPkgPath, inputPkgName string, buildRoot *os.Root) ([]string, error) { + inputPkgFS, closeFn, err := openPackageFS(inputPkgPath) + if err != nil { + return nil, fmt.Errorf("failed to open input package %q: %w", inputPkgPath, err) + } + defer closeFn() + + manifestBytes, err := fs.ReadFile(inputPkgFS, packages.PackageManifestFile) + if err != nil { + return nil, fmt.Errorf("failed to read input package manifest: %w", err) + } + manifest, err := packages.ReadPackageManifestBytes(manifestBytes) + if err != nil { + return nil, fmt.Errorf("failed to parse input package manifest: %w", err) + } + + seen := make(map[string]bool) + copiedNames := make([]string, 0) + for _, pt := range manifest.PolicyTemplates { + var names []string + switch { + case len(pt.TemplatePaths) > 0: + names = pt.TemplatePaths + case pt.TemplatePath != "": + names = []string{pt.TemplatePath} + } + for _, name := range names { + if seen[name] { + continue + } + seen[name] = true + // copy the template from "agent/input" directory of the input package to the "agent/input" directory of the build package + content, err := fs.ReadFile(inputPkgFS, path.Join("agent", "input", name)) + if err != nil { + return nil, fmt.Errorf("failed to read template %q from agent/input (declared in manifest): %w", name, err) + } + destName := inputPkgName + "-" + name + // create the agent/input directory if it doesn't exist + agentInputDir := path.Join("agent", "input") + if err := buildRoot.MkdirAll(agentInputDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create agent/input directory: %w", err) + } + destPath := path.Join(agentInputDir, destName) + if err := buildRoot.WriteFile(destPath, content, 0644); err != nil { + return nil, fmt.Errorf("failed to write template %q: %w", destName, err) + } + logger.Debugf("Copied input package template: %s -> %s", name, destName) + copiedNames = append(copiedNames, destName) + } + } + return copiedNames, nil +} + +// setInputPolicyTemplateTemplatePaths updates the manifest YAML document to set the template_paths for the specified policy template input to the provided paths +func setInputPolicyTemplateTemplatePaths(doc *yaml.Node, policyTemplatesIdx int, inputIdx int, paths []string) error { + // Navigate: document -> root mapping -> "policy_templates" -> sequence -> item [policyTemplatesIdx] -> mapping -> "inputs" -> sequence -> item [inputIdx] -> input mapping. + root := doc + if root.Kind == yaml.DocumentNode { + if len(root.Content) == 0 { + return fmt.Errorf("failed to set policy template input paths: empty YAML document") + } + root = root.Content[0] + } + if root.Kind != yaml.MappingNode { + return fmt.Errorf("failed to set policy template input paths: expected mapping node at document root") + } + + // policy_templates: + // - inputs: + // - template_path: foo + policyTemplatesNode := mappingValue(root, "policy_templates") + if policyTemplatesNode == nil { + return fmt.Errorf("failed to set policy template input paths: 'policy_templates' key not found in manifest") + } + if policyTemplatesNode.Kind != yaml.SequenceNode { + return fmt.Errorf("failed to set policy template input paths: 'policy_templates' is not a sequence") + } + if policyTemplatesIdx < 0 || policyTemplatesIdx >= len(policyTemplatesNode.Content) { + return fmt.Errorf("failed to set policy template input paths: policy template index %d out of range (len=%d)", policyTemplatesIdx, len(policyTemplatesNode.Content)) + } + + policyTemplateNode := policyTemplatesNode.Content[policyTemplatesIdx] + if policyTemplateNode.Kind != yaml.MappingNode { + return fmt.Errorf("failed to set policy template input paths: policy template entry %d is not a mapping", policyTemplatesIdx) + } + + inputsNode := mappingValue(policyTemplateNode, "inputs") + if inputsNode == nil { + return fmt.Errorf("failed to set policy template input paths: 'inputs' key not found in policy template %d", policyTemplatesIdx) + } + if inputsNode.Kind != yaml.SequenceNode { + return fmt.Errorf("failed to set policy template input paths: 'inputs' is not a sequence") + } + if inputIdx < 0 || inputIdx >= len(inputsNode.Content) { + return fmt.Errorf("failed to set policy template input paths: input index %d out of range (len=%d)", inputIdx, len(inputsNode.Content)) + } + + inputNode := inputsNode.Content[inputIdx] + if inputNode.Kind != yaml.MappingNode { + return fmt.Errorf("failed to set policy template input paths: input entry %d is not a mapping", inputIdx) + } + + // Remove singular template_path if present. + removeKey(inputNode, "template_path") + + // Build the template_paths sequence node. + seqNode := &yaml.Node{Kind: yaml.SequenceNode} + for _, p := range paths { + seqNode.Content = append(seqNode.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: p}) + } + + // Upsert template_paths on the input node. + upsertKey(inputNode, "template_paths", seqNode) + + return nil +} diff --git a/internal/requiredinputs/policytemplates_test.go b/internal/requiredinputs/policytemplates_test.go new file mode 100644 index 0000000000..e1f8de8d69 --- /dev/null +++ b/internal/requiredinputs/policytemplates_test.go @@ -0,0 +1,140 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package requiredinputs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-package/internal/packages" +) + +func TestBundlePolicyTemplatesInputPackageTemplates_InvalidYAML(t *testing.T) { + buildRootPath := t.TempDir() + buildRoot, err := os.OpenRoot(buildRootPath) + require.NoError(t, err) + defer buildRoot.Close() + + r := &RequiredInputsResolver{} + + manifestBytes := []byte("foo: [") + manifest, _ := packages.ReadPackageManifestBytes(manifestBytes) // may be nil/partial + + err = r.bundlePolicyTemplatesInputPackageTemplates(manifestBytes, manifest, nil, buildRoot) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse manifest YAML") +} + +// TestBundlePolicyTemplatesInputPackageTemplates_MultiplePolicyTemplates verifies that templates +// from ALL policy templates in an input package are bundled into agent/input/, not just the first +// one (Issue 5 in the alignment review). +func TestBundlePolicyTemplatesInputPackageTemplates_MultiplePolicyTemplates(t *testing.T) { + buildRootPath := t.TempDir() + buildRoot, err := os.OpenRoot(buildRootPath) + require.NoError(t, err) + defer buildRoot.Close() + + r := &RequiredInputsResolver{} + + manifestBytes := []byte(` +type: integration +requires: + input: + - package: sql + version: 0.1.0 +policy_templates: + - inputs: + - package: sql +`) + err = buildRoot.WriteFile("manifest.yml", manifestBytes, 0644) + require.NoError(t, err) + + manifest, err := packages.ReadPackageManifestBytes(manifestBytes) + require.NoError(t, err) + + fakeInputDir := createFakeInputWithMultiplePolicyTemplates(t) + inputPkgPaths := map[string]string{"sql": fakeInputDir} + + err = r.bundlePolicyTemplatesInputPackageTemplates(manifestBytes, manifest, inputPkgPaths, buildRoot) + require.NoError(t, err) + + // All templates from both policy templates in the input package must be present. + _, err = buildRoot.ReadFile(filepath.Join("agent", "input", "sql-input.yml.hbs")) + require.NoError(t, err, "template from first policy_template must be bundled") + _, err = buildRoot.ReadFile(filepath.Join("agent", "input", "sql-metrics.yml.hbs")) + require.NoError(t, err, "template from second policy_template must be bundled") + _, err = buildRoot.ReadFile(filepath.Join("agent", "input", "sql-extra.yml.hbs")) + require.NoError(t, err, "extra template from second policy_template must be bundled") + + updated, err := buildRoot.ReadFile("manifest.yml") + require.NoError(t, err) + updatedManifest, err := packages.ReadPackageManifestBytes(updated) + require.NoError(t, err) + require.Len(t, updatedManifest.PolicyTemplates, 1) + require.Len(t, updatedManifest.PolicyTemplates[0].Inputs, 1) + input := updatedManifest.PolicyTemplates[0].Inputs[0] + assert.Empty(t, input.TemplatePath) + assert.Equal(t, []string{"sql-input.yml.hbs", "sql-metrics.yml.hbs", "sql-extra.yml.hbs"}, input.TemplatePaths) +} + +func TestBundlePolicyTemplatesInputPackageTemplates_SuccessTemplatesCopied(t *testing.T) { + buildRootPath := t.TempDir() + buildRoot, err := os.OpenRoot(buildRootPath) + require.NoError(t, err) + defer buildRoot.Close() + + r := &RequiredInputsResolver{} + + // create current package manifest with one policy template input referencing an input package template + // it has an existing template, so both the existing and input package template should be copied and the manifest updated to reference both + manifestBytes := []byte(` +type: integration +requires: + input: + - package: sql + version: 0.1.0 +policy_templates: + - inputs: + - package: sql + template_path: existing.yml.hbs +`) + err = buildRoot.WriteFile("manifest.yml", manifestBytes, 0644) + require.NoError(t, err) + err = buildRoot.MkdirAll(filepath.Join("agent", "input"), 0755) + require.NoError(t, err) + err = buildRoot.WriteFile(filepath.Join("agent", "input", "existing.yml.hbs"), []byte("existing content"), 0644) + require.NoError(t, err) + + // parse manifest to pass to function + manifest, err := packages.ReadPackageManifestBytes(manifestBytes) + require.NoError(t, err) + + fakeInputDir := createFakeInputHelper(t) + inputPkgPaths := map[string]string{"sql": fakeInputDir} + + err = r.bundlePolicyTemplatesInputPackageTemplates(manifestBytes, manifest, inputPkgPaths, buildRoot) + require.NoError(t, err) + + // Files exist. + _, err = buildRoot.ReadFile(filepath.Join("agent", "input", "sql-input.yml.hbs")) + require.NoError(t, err) + _, err = buildRoot.ReadFile(filepath.Join("agent", "input", "existing.yml.hbs")) + require.NoError(t, err) + + // Written manifest has template_paths set and template_path removed for that input. + updated, err := buildRoot.ReadFile("manifest.yml") + require.NoError(t, err) + updatedManifest, err := packages.ReadPackageManifestBytes(updated) + require.NoError(t, err) + require.Len(t, updatedManifest.PolicyTemplates, 1) + require.Len(t, updatedManifest.PolicyTemplates[0].Inputs, 1) + input := updatedManifest.PolicyTemplates[0].Inputs[0] + assert.Empty(t, input.TemplatePath) + assert.Equal(t, []string{"sql-input.yml.hbs", "existing.yml.hbs"}, input.TemplatePaths) +} diff --git a/internal/requiredinputs/requiredinputs.go b/internal/requiredinputs/requiredinputs.go new file mode 100644 index 0000000000..9271e8c596 --- /dev/null +++ b/internal/requiredinputs/requiredinputs.go @@ -0,0 +1,155 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package requiredinputs + +import ( + "archive/zip" + "errors" + "fmt" + "io/fs" + "os" + "path" + + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/packages" +) + +type eprClient interface { + DownloadPackage(packageName string, packageVersion string, tmpDir string) (string, error) +} + +// Resolver bundles required input package templates into a built package tree. +type Resolver interface { + BundleInputPackageTemplates(buildPackageRoot string) error +} + +// NoopRequiredInputsResolver is a no-op implementation of Resolver. +// TODO: Replace with a resolver that supports test overrides (e.g. local package paths) +// when implementing local input package resolution for development and testing workflows. +type NoopRequiredInputsResolver struct{} + +func (r *NoopRequiredInputsResolver) BundleInputPackageTemplates(_ string) error { + return nil +} + +// RequiredInputsResolver is a helper for resolving required input packages. +type RequiredInputsResolver struct { + eprClient eprClient +} + +// NewRequiredInputsResolver returns a Resolver that downloads required input packages from the registry. +func NewRequiredInputsResolver(eprClient eprClient) (*RequiredInputsResolver, error) { + return &RequiredInputsResolver{ + eprClient: eprClient, + }, nil +} + +func (r *RequiredInputsResolver) BundleInputPackageTemplates(buildPackageRoot string) error { + + buildRoot, err := os.OpenRoot(buildPackageRoot) + if err != nil { + return fmt.Errorf("failed to open build package root: %w", err) + } + defer buildRoot.Close() + + manifestBytes, err := buildRoot.ReadFile("manifest.yml") + if err != nil { + return fmt.Errorf("failed to read package manifest: %w", err) + } + manifest, err := packages.ReadPackageManifestBytes(manifestBytes) + if err != nil { + return fmt.Errorf("failed to parse package manifest: %w", err) + } + + // validate that the package is an integration and has required input packages + if manifest.Type != "integration" { + return nil + } + if manifest.Requires == nil || len(manifest.Requires.Input) == 0 { + logger.Debug("Package has no required input packages, skipping template bundling") + return nil + } + + tmpDir, err := os.MkdirTemp("", "elastic-package-input-pkgs-*") + if err != nil { + return fmt.Errorf("failed to create temp directory for input packages: %w", err) + } + defer os.RemoveAll(tmpDir) + + inputPkgPaths, err := r.mapRequiredInputPackagesPaths(manifest.Requires.Input, tmpDir) + if err != nil { + return err + } + + if err := r.bundlePolicyTemplatesInputPackageTemplates(manifestBytes, manifest, inputPkgPaths, buildRoot); err != nil { + return fmt.Errorf("failed to bundle policy template input package templates: %w", err) + } + + if err := r.bundleDataStreamTemplates(inputPkgPaths, buildRoot); err != nil { + return fmt.Errorf("failed to bundle data stream input package templates: %w", err) + } + + return nil +} + +// downloadInputsToTmp downloads required input packages to the temporary directory. +// It returns a map of package name to zip path. +func (r *RequiredInputsResolver) mapRequiredInputPackagesPaths(manifestInputRequires []packages.PackageDependency, tmpDir string) (map[string]string, error) { + inputPkgPaths := make(map[string]string, len(manifestInputRequires)) + errs := make([]error, 0, len(manifestInputRequires)) + for _, inputDependency := range manifestInputRequires { + if _, ok := inputPkgPaths[inputDependency.Package]; ok { + // skip if already downloaded + continue + } + path, err := r.eprClient.DownloadPackage(inputDependency.Package, inputDependency.Version, tmpDir) + if err != nil { + // all required input packages must be downloaded successfully + errs = append(errs, fmt.Errorf("failed to download input package %q: %w", inputDependency.Package, err)) + continue + } + + // key is package name, for now we only support one version per package + inputPkgPaths[inputDependency.Package] = path + logger.Debugf("Resolved input package %q at %s", inputDependency.Package, path) + } + + return inputPkgPaths, errors.Join(errs...) +} + +// openPackageFS returns an fs.FS rooted at the package root (manifest.yml at +// the top level) and a close function that must be called when done. For +// directory packages it closes the os.Root; for zip packages it closes the +// underlying zip.ReadCloser. +func openPackageFS(pkgPath string) (fs.FS, func() error, error) { + info, err := os.Stat(pkgPath) + if err != nil { + return nil, nil, err + } + if info.IsDir() { + // open the package directory as a root + root, err := os.OpenRoot(pkgPath) + if err != nil { + return nil, nil, err + } + return root.FS(), root.Close, nil + } + // open the package zip as a zip reader + zipRC, err := zip.OpenReader(pkgPath) + if err != nil { + return nil, nil, err + } + matched, err := fs.Glob(zipRC, "*/"+packages.PackageManifestFile) + if err != nil || len(matched) == 0 { + zipRC.Close() + return nil, nil, fmt.Errorf("failed to find package manifest in zip %s", pkgPath) + } + subFS, err := fs.Sub(zipRC, path.Dir(matched[0])) + if err != nil { + zipRC.Close() + return nil, nil, err + } + return subFS, zipRC.Close, nil +} diff --git a/internal/requiredinputs/requiredinputs_test.go b/internal/requiredinputs/requiredinputs_test.go new file mode 100644 index 0000000000..5ee5ddaa49 --- /dev/null +++ b/internal/requiredinputs/requiredinputs_test.go @@ -0,0 +1,146 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package requiredinputs + +import ( + "fmt" + "os" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-package/internal/packages" +) + +type fakeEprClient struct { + downloadPackageFunc func(packageName string, packageVersion string, tmpDir string) (string, error) +} + +func (f *fakeEprClient) DownloadPackage(packageName string, packageVersion string, tmpDir string) (string, error) { + if f.downloadPackageFunc != nil { + return f.downloadPackageFunc(packageName, packageVersion, tmpDir) + } + return "", fmt.Errorf("download package not implemented") +} + +func TestBundleInputPackageTemplates_Success(t *testing.T) { + fakeInputPath := createFakeInputHelper(t) + fakeEprClient := &fakeEprClient{ + downloadPackageFunc: func(packageName string, packageVersion string, tmpDir string) (string, error) { + return fakeInputPath, nil + }, + } + buildPackageRoot := t.TempDir() + + manifest := []byte(`name: test-package +version: 0.1.0 +type: integration +requires: + input: + - package: sql + version: 0.1.0 +policy_templates: + - inputs: + - package: sql + - type: logs +`) + err := os.WriteFile(path.Join(buildPackageRoot, "manifest.yml"), manifest, 0644) + require.NoError(t, err) + + resolver, err := NewRequiredInputsResolver(fakeEprClient) + require.NoError(t, err) + + err = resolver.BundleInputPackageTemplates(buildPackageRoot) + require.NoError(t, err) + + _, err = os.ReadFile(path.Join(buildPackageRoot, "agent", "input", "sql-input.yml.hbs")) + require.NoError(t, err) + + updatedManifestBytes, err := os.ReadFile(path.Join(buildPackageRoot, "manifest.yml")) + require.NoError(t, err) + updatedManifest, err := packages.ReadPackageManifestBytes(updatedManifestBytes) + require.NoError(t, err) + require.Len(t, updatedManifest.Requires.Input, 1) + require.Equal(t, "sql", updatedManifest.Requires.Input[0].Package) + require.Equal(t, "0.1.0", updatedManifest.Requires.Input[0].Version) + + require.Equal(t, "sql", updatedManifest.PolicyTemplates[0].Inputs[0].Package) + require.Len(t, updatedManifest.PolicyTemplates[0].Inputs[0].TemplatePaths, 1) + require.Equal(t, "sql-input.yml.hbs", updatedManifest.PolicyTemplates[0].Inputs[0].TemplatePaths[0]) + +} + +func TestBundleInputPackageTemplates_NoManifest(t *testing.T) { + fakeInputPath := createFakeInputHelper(t) + fakeEprClient := &fakeEprClient{ + downloadPackageFunc: func(packageName string, packageVersion string, tmpDir string) (string, error) { + return fakeInputPath, nil + }, + } + buildPackageRoot := t.TempDir() + + resolver, err := NewRequiredInputsResolver(fakeEprClient) + require.NoError(t, err) + + err = resolver.BundleInputPackageTemplates(buildPackageRoot) + require.Error(t, err) + assert.ErrorContains(t, err, "failed to read package manifest") +} + +func TestBundleInputPackageTemplates_SkipNoIntegration(t *testing.T) { + fakeInputPath := createFakeInputHelper(t) + fakeEprClient := &fakeEprClient{ + downloadPackageFunc: func(packageName string, packageVersion string, tmpDir string) (string, error) { + return fakeInputPath, nil + }, + } + buildPackageRoot := t.TempDir() + + manifest := []byte(`name: test-package +version: 0.1.0 +type: input +`) + err := os.WriteFile(path.Join(buildPackageRoot, "manifest.yml"), manifest, 0644) + require.NoError(t, err) + + resolver, err := NewRequiredInputsResolver(fakeEprClient) + require.NoError(t, err) + + err = resolver.BundleInputPackageTemplates(buildPackageRoot) + require.NoError(t, err) +} + +func TestBundleInputPackageTemplates_NoRequires(t *testing.T) { + fakeEprClient := &fakeEprClient{ + downloadPackageFunc: func(packageName string, packageVersion string, tmpDir string) (string, error) { + return "", fmt.Errorf("no download without requires") + }, + } + buildPackageRoot := t.TempDir() + + manifest := []byte(`name: test-package +version: 0.1.0 +type: integration +policy_templates: + - inputs: + - type: logs +`) + err := os.WriteFile(path.Join(buildPackageRoot, "manifest.yml"), manifest, 0644) + require.NoError(t, err) + + resolver, err := NewRequiredInputsResolver(fakeEprClient) + require.NoError(t, err) + + err = resolver.BundleInputPackageTemplates(buildPackageRoot) + require.NoError(t, err) + + updatedManifestBytes, err := os.ReadFile(path.Join(buildPackageRoot, "manifest.yml")) + require.NoError(t, err) + updatedManifest, err := packages.ReadPackageManifestBytes(updatedManifestBytes) + require.NoError(t, err) + require.Nil(t, updatedManifest.Requires) +} diff --git a/internal/requiredinputs/streams.go b/internal/requiredinputs/streams.go new file mode 100644 index 0000000000..b9d5410126 --- /dev/null +++ b/internal/requiredinputs/streams.go @@ -0,0 +1,201 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package requiredinputs + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path" + + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/packages" +) + +func (r *RequiredInputsResolver) bundleDataStreamTemplates(inputPkgPaths map[string]string, buildRoot *os.Root) error { + // get all data stream manifest paths in the build package + dsManifestsPaths, err := fs.Glob(buildRoot.FS(), "data_stream/*/manifest.yml") + if err != nil { + return fmt.Errorf("failed to glob data stream manifests: %w", err) + } + + errorList := make([]error, 0) + for _, manifestPath := range dsManifestsPaths { + manifestBytes, err := buildRoot.ReadFile(manifestPath) + if err != nil { + return fmt.Errorf("failed to read data stream manifest %q: %w", manifestPath, err) + } + // parse the manifest YAML document preserving formatting for targeted modifications + // using manifestBytes allows us to preserve comments and formatting in the manifest when we update it with template paths from input packages + var doc yaml.Node + if err := yaml.Unmarshal(manifestBytes, &doc); err != nil { + return fmt.Errorf("failed to parse data stream manifest YAML: %w", err) + } + + manifest, err := packages.ReadDataStreamManifestBytes(manifestBytes) + if err != nil { + return fmt.Errorf("failed to parse data stream manifest %q: %w", manifestPath, err) + } + for idx, stream := range manifest.Streams { + if stream.Package == "" { + continue + } + pkgPath, ok := inputPkgPaths[stream.Package] + if !ok { + errorList = append(errorList, fmt.Errorf("failed to resolve input package %q for stream in manifest %q: not listed in requires.input", stream.Package, manifestPath)) + continue + } + dsRootDir := path.Dir(manifestPath) + inputPaths, err := r.collectAndCopyInputPkgDataStreams(dsRootDir, pkgPath, stream.Package, buildRoot) + if err != nil { + return fmt.Errorf("failed to collect and copy input package data stream templates for manifest %q: %w", manifestPath, err) + } + if len(inputPaths) == 0 { + continue + } + + // current manifest template paths + paths := make([]string, 0) + // if composable package has included custom template path or paths, include them + // if no template paths are included at the manifest, only the imported templates are included + if stream.TemplatePath != "" { + paths = append(paths, stream.TemplatePath) + } else if len(stream.TemplatePaths) > 0 { + paths = append(paths, stream.TemplatePaths...) + } + paths = append(inputPaths, paths...) + + if err := setStreamTemplatePaths(&doc, idx, paths); err != nil { + return fmt.Errorf("failed to set stream template paths in manifest %q: %w", manifestPath, err) + } + + } + + // Serialise the updated YAML document back to disk. + updated, err := formatYAMLNode(&doc) + if err != nil { + return fmt.Errorf("failed to format updated manifest: %w", err) + } + if err := buildRoot.WriteFile(manifestPath, updated, 0664); err != nil { + return fmt.Errorf("failed to write updated manifest: %w", err) + } + + } + return errors.Join(errorList...) +} + +// collectAndCopyInputPkgDataStreams collects the data streams from the input package and copies them to the agent/input directory of the build package +// it returns the list of copied data stream names +// +// Design note: input package templates are authored for input-level compilation, where available +// variables are: package vars + input.vars. When these templates are copied to the integration's +// data_stream//agent/stream/ directory and compiled as stream templates, Fleet compiles them +// with package vars + input.vars + stream.vars. For templates that only reference package-level +// or input-level variables this works correctly. However, stream-level vars defined on the +// integration's data stream will NOT be accessible from input package templates — the template +// content must explicitly reference them. If stream-level vars need to be rendered, add an +// integration-owned stream template and include it after the input package templates in +// template_paths (integration templates are appended last and take precedence). +// See https://github.com/elastic/elastic-package/issues/3279 for the follow-up work on +// merging variable definitions from input packages and composable packages at build time. +func (r *RequiredInputsResolver) collectAndCopyInputPkgDataStreams(dsRootDir, inputPkgPath, inputPkgName string, buildRoot *os.Root) ([]string, error) { + inputPkgFS, closeFn, err := openPackageFS(inputPkgPath) + if err != nil { + return nil, fmt.Errorf("failed to open input package %q: %w", inputPkgPath, err) + } + defer closeFn() + + manifestBytes, err := fs.ReadFile(inputPkgFS, "manifest.yml") + if err != nil { + return nil, fmt.Errorf("failed to read input package manifest: %w", err) + } + manifest, err := packages.ReadPackageManifestBytes(manifestBytes) + if err != nil { + return nil, fmt.Errorf("failed to parse input package manifest: %w", err) + } + + seen := make(map[string]bool) + copiedNames := make([]string, 0) + for _, pt := range manifest.PolicyTemplates { + var names []string + switch { + case len(pt.TemplatePaths) > 0: + names = pt.TemplatePaths + case pt.TemplatePath != "": + names = []string{pt.TemplatePath} + } + for _, name := range names { + if seen[name] { + continue + } + seen[name] = true + // copy the template from "agent/input" directory of the input package to the "agent/stream" directory of the build package + content, err := fs.ReadFile(inputPkgFS, path.Join("agent", "input", name)) + if err != nil { + return nil, fmt.Errorf("failed to read template %q from agent/input (declared in manifest): %w", name, err) + } + destName := inputPkgName + "-" + name + // create the agent/stream directory if it doesn't exist + agentStreamDir := path.Join(dsRootDir, "agent", "stream") + if err := buildRoot.MkdirAll(agentStreamDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create agent/stream directory: %w", err) + } + destPath := path.Join(agentStreamDir, destName) + if err := buildRoot.WriteFile(destPath, content, 0644); err != nil { + return nil, fmt.Errorf("failed to write template %q: %w", destName, err) + } + logger.Debugf("Copied input package template: %s -> %s", name, destName) + copiedNames = append(copiedNames, destName) + } + } + return copiedNames, nil +} + +func setStreamTemplatePaths(doc *yaml.Node, streamIdx int, paths []string) error { + // Navigate: document -> mapping -> "streams" key -> sequence -> item [streamIdx] + root := doc + if root.Kind == yaml.DocumentNode { + if len(root.Content) == 0 { + return fmt.Errorf("failed to set stream template paths: empty YAML document") + } + root = root.Content[0] + } + if root.Kind != yaml.MappingNode { + return fmt.Errorf("failed to set stream template paths: expected mapping node at document root") + } + + streamsNode := mappingValue(root, "streams") + if streamsNode == nil { + return fmt.Errorf("failed to set stream template paths: 'streams' key not found in manifest") + } + if streamsNode.Kind != yaml.SequenceNode { + return fmt.Errorf("failed to set stream template paths: 'streams' is not a sequence") + } + if streamIdx >= len(streamsNode.Content) { + return fmt.Errorf("failed to set stream template paths: stream index %d out of range (len=%d)", streamIdx, len(streamsNode.Content)) + } + + streamNode := streamsNode.Content[streamIdx] + if streamNode.Kind != yaml.MappingNode { + return fmt.Errorf("failed to set stream template paths: stream entry %d is not a mapping", streamIdx) + } + + // Remove singular template_path if present. + removeKey(streamNode, "template_path") + + // Build the template_paths sequence node. + seqNode := &yaml.Node{Kind: yaml.SequenceNode} + for _, p := range paths { + seqNode.Content = append(seqNode.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: p}) + } + + // Upsert template_paths. + upsertKey(streamNode, "template_paths", seqNode) + + return nil +} diff --git a/internal/requiredinputs/streams_test.go b/internal/requiredinputs/streams_test.go new file mode 100644 index 0000000000..f75b99e32d --- /dev/null +++ b/internal/requiredinputs/streams_test.go @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package requiredinputs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-package/internal/packages" +) + +// TestBundleDataStreamTemplates_MultiplePolicyTemplates verifies that templates from ALL +// policy templates in the input package are bundled, not just the first one (Issue 5). +func TestBundleDataStreamTemplates_MultiplePolicyTemplates(t *testing.T) { + buildRootPath := t.TempDir() + buildRoot, err := os.OpenRoot(buildRootPath) + require.NoError(t, err) + defer buildRoot.Close() + + r := &RequiredInputsResolver{} + + datastreamDir := filepath.Join("data_stream", "test_ds") + err = buildRoot.MkdirAll(datastreamDir, 0755) + require.NoError(t, err) + + manifestBytes := []byte(` +streams: + - package: sql +`) + err = buildRoot.WriteFile(filepath.Join(datastreamDir, "manifest.yml"), manifestBytes, 0644) + require.NoError(t, err) + + fakeInputDir := createFakeInputWithMultiplePolicyTemplates(t) + inputPkgPaths := map[string]string{"sql": fakeInputDir} + + err = r.bundleDataStreamTemplates(inputPkgPaths, buildRoot) + require.NoError(t, err) + + // All templates from both policy templates must be present. + _, err = buildRoot.ReadFile(filepath.Join(datastreamDir, "agent", "stream", "sql-input.yml.hbs")) + require.NoError(t, err, "template from first policy_template must be bundled") + _, err = buildRoot.ReadFile(filepath.Join(datastreamDir, "agent", "stream", "sql-metrics.yml.hbs")) + require.NoError(t, err, "template from second policy_template must be bundled") + _, err = buildRoot.ReadFile(filepath.Join(datastreamDir, "agent", "stream", "sql-extra.yml.hbs")) + require.NoError(t, err, "extra template from second policy_template must be bundled") + + updated, err := buildRoot.ReadFile(filepath.Join(datastreamDir, "manifest.yml")) + require.NoError(t, err) + updatedManifest, err := packages.ReadDataStreamManifestBytes(updated) + require.NoError(t, err) + require.Len(t, updatedManifest.Streams, 1) + assert.Equal(t, []string{"sql-input.yml.hbs", "sql-metrics.yml.hbs", "sql-extra.yml.hbs"}, updatedManifest.Streams[0].TemplatePaths) +} + +func TestBundleDataStreamTemplates_SuccessTemplatesCopied(t *testing.T) { + buildRootPath := t.TempDir() + buildRoot, err := os.OpenRoot(buildRootPath) + require.NoError(t, err) + defer buildRoot.Close() + + r := &RequiredInputsResolver{} + + datastreamDir := filepath.Join("data_stream", "test_ds") + err = buildRoot.MkdirAll(datastreamDir, 0755) + require.NoError(t, err) + // create current package manifest with one data stream input referencing an input package template + // it has an existing template, so both the existing and input package template should be copied and the manifest updated to reference both + manifestBytes := []byte(` +streams: + - package: sql + template_path: existing.yml.hbs +`) + err = buildRoot.WriteFile(filepath.Join(datastreamDir, "manifest.yml"), manifestBytes, 0644) + require.NoError(t, err) + err = buildRoot.MkdirAll(filepath.Join(datastreamDir, "agent", "stream"), 0755) + require.NoError(t, err) + err = buildRoot.WriteFile(filepath.Join(datastreamDir, "agent", "stream", "existing.yml.hbs"), []byte("existing content"), 0644) + require.NoError(t, err) + + fakeInputDir := createFakeInputHelper(t) + inputPkgPaths := map[string]string{"sql": fakeInputDir} + + err = r.bundleDataStreamTemplates(inputPkgPaths, buildRoot) + require.NoError(t, err) + + // Files exist. + _, err = buildRoot.ReadFile(filepath.Join(datastreamDir, "agent", "stream", "sql-input.yml.hbs")) + require.NoError(t, err) + _, err = buildRoot.ReadFile(filepath.Join(datastreamDir, "agent", "stream", "existing.yml.hbs")) + require.NoError(t, err) + + // Written manifest has template_paths set and template_path removed for that input. + updated, err := buildRoot.ReadFile(filepath.Join(datastreamDir, "manifest.yml")) + require.NoError(t, err) + updatedManifest, err := packages.ReadDataStreamManifestBytes(updated) + require.NoError(t, err) + require.Len(t, updatedManifest.Streams, 1) + input := updatedManifest.Streams[0] + assert.Empty(t, input.TemplatePath) + assert.Equal(t, []string{"sql-input.yml.hbs", "existing.yml.hbs"}, input.TemplatePaths) +} + +// TestBundleDataStreamTemplates_BundlesWithoutDataStreamsAssociation verifies that a data stream +// stream entry with package: X IS bundled even when the root policy template has no data_streams +// field. Bundling is driven solely by the data stream manifest's streams[].package reference. +func TestBundleDataStreamTemplates_BundlesWithoutDataStreamsAssociation(t *testing.T) { + buildRootPath := t.TempDir() + buildRoot, err := os.OpenRoot(buildRootPath) + require.NoError(t, err) + defer buildRoot.Close() + + r := &RequiredInputsResolver{} + + datastreamDir := filepath.Join("data_stream", "test_ds") + err = buildRoot.MkdirAll(datastreamDir, 0755) + require.NoError(t, err) + + manifestBytes := []byte(` +streams: + - package: sql +`) + err = buildRoot.WriteFile(filepath.Join(datastreamDir, "manifest.yml"), manifestBytes, 0644) + require.NoError(t, err) + + fakeInputDir := createFakeInputHelper(t) + inputPkgPaths := map[string]string{"sql": fakeInputDir} + + err = r.bundleDataStreamTemplates(inputPkgPaths, buildRoot) + require.NoError(t, err) + + // Template must be bundled even without a data_streams association in the root manifest. + _, err = buildRoot.ReadFile(filepath.Join(datastreamDir, "agent", "stream", "sql-input.yml.hbs")) + require.NoError(t, err, "template must be bundled when stream references an input package, regardless of data_streams field") + + // The data stream manifest must have template_paths set. + updated, err := buildRoot.ReadFile(filepath.Join(datastreamDir, "manifest.yml")) + require.NoError(t, err) + updatedManifest, err := packages.ReadDataStreamManifestBytes(updated) + require.NoError(t, err) + require.Len(t, updatedManifest.Streams, 1) + assert.Equal(t, []string{"sql-input.yml.hbs"}, updatedManifest.Streams[0].TemplatePaths) +} diff --git a/internal/requiredinputs/testhelpers_test.go b/internal/requiredinputs/testhelpers_test.go new file mode 100644 index 0000000000..fb5a4d8246 --- /dev/null +++ b/internal/requiredinputs/testhelpers_test.go @@ -0,0 +1,67 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package requiredinputs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func createFakeInputHelper(t *testing.T) string { + t.Helper() + // create fake input package with manifest and template file + fakeDownloadedPkgDir := t.TempDir() + inputPkgDir := filepath.Join(fakeDownloadedPkgDir, "sql") + err := os.Mkdir(inputPkgDir, 0755) + require.NoError(t, err) + inputManifestBytes := []byte(`name: sql +version: 0.1.0 +type: input +policy_templates: + - input: sql + template_path: input.yml.hbs +`) + err = os.WriteFile(filepath.Join(inputPkgDir, "manifest.yml"), inputManifestBytes, 0644) + require.NoError(t, err) + err = os.MkdirAll(filepath.Join(inputPkgDir, "agent", "input"), 0755) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(inputPkgDir, "agent", "input", "input.yml.hbs"), []byte("template content"), 0644) + require.NoError(t, err) + return inputPkgDir +} + +func createFakeInputWithMultiplePolicyTemplates(t *testing.T) string { + t.Helper() + fakeDownloadedPkgDir := t.TempDir() + inputPkgDir := filepath.Join(fakeDownloadedPkgDir, "sql") + err := os.Mkdir(inputPkgDir, 0755) + require.NoError(t, err) + // Input package with two policy templates, each declaring a distinct template. + inputManifestBytes := []byte(`name: sql +version: 0.1.0 +type: input +policy_templates: + - input: sql + template_path: input.yml.hbs + - input: sql/metrics + template_paths: + - metrics.yml.hbs + - extra.yml.hbs +`) + err = os.WriteFile(filepath.Join(inputPkgDir, "manifest.yml"), inputManifestBytes, 0644) + require.NoError(t, err) + err = os.MkdirAll(filepath.Join(inputPkgDir, "agent", "input"), 0755) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(inputPkgDir, "agent", "input", "input.yml.hbs"), []byte("input template"), 0644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(inputPkgDir, "agent", "input", "metrics.yml.hbs"), []byte("metrics template"), 0644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(inputPkgDir, "agent", "input", "extra.yml.hbs"), []byte("extra template"), 0644) + require.NoError(t, err) + return inputPkgDir +} diff --git a/internal/requiredinputs/yamlutil.go b/internal/requiredinputs/yamlutil.go new file mode 100644 index 0000000000..e5ddd54110 --- /dev/null +++ b/internal/requiredinputs/yamlutil.go @@ -0,0 +1,62 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package requiredinputs + +import ( + "fmt" + "slices" + + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-package/internal/formatter" +) + +// mappingValue returns the value node for the given key in a YAML mapping node, +// or nil if the key is not present. +func mappingValue(node *yaml.Node, key string) *yaml.Node { + idx := slices.IndexFunc(node.Content, func(n *yaml.Node) bool { + return n.Value == key + }) + if idx < 0 || idx+1 >= len(node.Content) { + return nil + } + return node.Content[idx+1] +} + +// removeKey removes a key-value pair from a YAML mapping node. +func removeKey(node *yaml.Node, key string) { + idx := slices.IndexFunc(node.Content, func(n *yaml.Node) bool { + return n.Value == key + }) + if idx >= 0 && idx+1 < len(node.Content) { + node.Content = slices.Delete(node.Content, idx, idx+2) + } +} + +// upsertKey sets key to value in a YAML mapping node, adding it if absent. +func upsertKey(node *yaml.Node, key string, value *yaml.Node) { + idx := slices.IndexFunc(node.Content, func(n *yaml.Node) bool { + return n.Value == key + }) + if idx >= 0 && idx+1 < len(node.Content) { + node.Content[idx+1] = value + return + } + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} + node.Content = append(node.Content, keyNode, value) +} + +func formatYAMLNode(doc *yaml.Node) ([]byte, error) { + raw, err := yaml.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("failed to marshal YAML: %w", err) + } + yamlFormatter := formatter.NewYAMLFormatter(formatter.KeysWithDotActionNone) + formatted, _, err := yamlFormatter.Format(raw) + if err != nil { + return nil, fmt.Errorf("failed to format YAML: %w", err) + } + return formatted, nil +} diff --git a/internal/resources/fleetpackage.go b/internal/resources/fleetpackage.go index e5c17e96ff..408fb74f88 100644 --- a/internal/resources/fleetpackage.go +++ b/internal/resources/fleetpackage.go @@ -17,6 +17,7 @@ import ( "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/packages/installer" + "github.com/elastic/elastic-package/internal/requiredinputs" ) type FleetPackage struct { @@ -38,6 +39,9 @@ type FleetPackage struct { // Force forces operations, as reinstalling a package that seems to // be already installed. Force bool + + // RequiredInputsResolver is the resolver for required input packages. + RequiredInputsResolver requiredinputs.Resolver } func (f *FleetPackage) String() string { @@ -64,11 +68,12 @@ func (f *FleetPackage) installer(ctx resource.Context) (installer.Installer, err } return installer.NewForPackage(installer.Options{ - Kibana: provider.Client, - PackageRoot: f.PackageRoot, - SkipValidation: true, - RepositoryRoot: f.RepositoryRoot, - SchemaURLs: f.SchemaURLs, + Kibana: provider.Client, + PackageRoot: f.PackageRoot, + SkipValidation: true, + RepositoryRoot: f.RepositoryRoot, + SchemaURLs: f.SchemaURLs, + RequiredInputsResolver: f.RequiredInputsResolver, }) } diff --git a/internal/resources/fleetpackage_test.go b/internal/resources/fleetpackage_test.go index f0bb905475..bdcd5b1e50 100644 --- a/internal/resources/fleetpackage_test.go +++ b/internal/resources/fleetpackage_test.go @@ -29,8 +29,9 @@ func TestRequiredProvider(t *testing.T) { _, err = manager.Apply(resource.Resources{ &FleetPackage{ - PackageRoot: "../../test/packages/parallel/nginx", - RepositoryRoot: repositoryRoot, + PackageRoot: "../../test/packages/parallel/nginx", + RepositoryRoot: repositoryRoot, + RequiredInputsResolver: &requiredInputsResolverMock{}, }, }) if assert.Error(t, err) { @@ -38,6 +39,14 @@ func TestRequiredProvider(t *testing.T) { } } +type requiredInputsResolverMock struct { + BundleInputPackageTemplatesFunc func(buildPackageRoot string) error +} + +func (r *requiredInputsResolverMock) BundleInputPackageTemplates(buildPackageRoot string) error { + return nil +} + func TestPackageLifecycle(t *testing.T) { cases := []struct { title string @@ -62,9 +71,10 @@ func TestPackageLifecycle(t *testing.T) { packageRoot := filepath.Join(repositoryRoot.Name(), "test", "packages", "parallel", c.name) fleetPackage := FleetPackage{ - PackageRoot: packageRoot, - RepositoryRoot: repositoryRoot, - SchemaURLs: fields.NewSchemaURLs(), + PackageRoot: packageRoot, + RepositoryRoot: repositoryRoot, + SchemaURLs: fields.NewSchemaURLs(), + RequiredInputsResolver: &requiredInputsResolverMock{}, } manager := resource.NewManager() manager.RegisterProvider(DefaultKibanaProviderName, &KibanaProvider{Client: kibanaClient}) diff --git a/internal/resources/fleetpolicy_test.go b/internal/resources/fleetpolicy_test.go index c0b93b0078..c57560de70 100644 --- a/internal/resources/fleetpolicy_test.go +++ b/internal/resources/fleetpolicy_test.go @@ -122,10 +122,11 @@ func withPackageResources(agentPolicy *FleetAgentPolicy, repostoryRoot *os.Root) var resources resource.Resources for _, policy := range agentPolicy.PackagePolicies { resources = append(resources, &FleetPackage{ - PackageRoot: policy.PackageRoot, - Absent: agentPolicy.Absent, - RepositoryRoot: repostoryRoot, - SchemaURLs: fields.NewSchemaURLs(), + PackageRoot: policy.PackageRoot, + Absent: agentPolicy.Absent, + RepositoryRoot: repostoryRoot, + SchemaURLs: fields.NewSchemaURLs(), + RequiredInputsResolver: &requiredInputsResolverMock{}, }) } return append(resources, agentPolicy) diff --git a/internal/stack/environment.go b/internal/stack/environment.go index 26e85fd3ba..50a538a0f4 100644 --- a/internal/stack/environment.go +++ b/internal/stack/environment.go @@ -154,7 +154,7 @@ func (p *environmentProvider) initClients(appConfig *install.ApplicationConfigur } p.elasticsearch = elasticsearch - p.registry = registry.NewClient(packageRegistryBaseURL(p.profile, appConfig)) + p.registry = registry.NewClient(PackageRegistryBaseURL(p.profile, appConfig)) return nil } diff --git a/internal/stack/registry.go b/internal/stack/registry.go index 4314389357..b1e8cea490 100644 --- a/internal/stack/registry.go +++ b/internal/stack/registry.go @@ -5,6 +5,10 @@ package stack import ( + "net/url" + "os" + "strings" + "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/registry" @@ -28,10 +32,10 @@ func packageRegistryProxyToURL(profile *profile.Profile, appConfig *install.Appl return registry.ProductionURL } -// packageRegistryBaseURL returns the package registry base URL to be used, considering +// PackageRegistryBaseURL returns the package registry base URL to be used, considering // profile settings and application configuration. The priority is given to // profile settings over application configuration. -func packageRegistryBaseURL(profile *profile.Profile, appConfig *install.ApplicationConfiguration) string { +func PackageRegistryBaseURL(profile *profile.Profile, appConfig *install.ApplicationConfiguration) string { if registryURL := profile.Config(configElasticEPRURL, ""); registryURL != "" { return registryURL } @@ -42,3 +46,28 @@ func packageRegistryBaseURL(profile *profile.Profile, appConfig *install.Applica } return registry.ProductionURL } + +// RegistryClientOptions returns TLS options for the registry client so it works +// with the elastic-package stack (same CA as Kibana/ES) or local HTTPS registries. +// Profile may be nil (e.g. in build); then only CACertificateEnv is used for CA. +func RegistryClientOptions(registryBaseURL string, profile *profile.Profile) []registry.ClientOption { + var opts []registry.ClientOption + caPath := os.Getenv(CACertificateEnv) + if caPath == "" && profile != nil { + caPath, _ = FindCACertificate(profile) + } + if caPath != "" { + if _, err := os.Stat(caPath); err == nil { + opts = append(opts, registry.CertificateAuthority(caPath)) + return opts + } + } + u, err := url.Parse(registryBaseURL) + if err != nil { + return opts + } + if u.Scheme == "https" && (strings.ToLower(u.Hostname()) == "localhost" || u.Hostname() == "127.0.0.1") { + opts = append(opts, registry.TLSSkipVerify()) + } + return opts +} diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 4ccdc96ce2..2d5ca6cf74 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -232,7 +232,7 @@ func (sp *serverlessProvider) createClients(project *serverless.Project, appConf return fmt.Errorf("failed to create kibana client: %w", err) } - sp.registryClient = registry.NewClient(packageRegistryBaseURL(sp.profile, appConfig)) + sp.registryClient = registry.NewClient(PackageRegistryBaseURL(sp.profile, appConfig)) return nil } diff --git a/internal/testrunner/runners/asset/tester.go b/internal/testrunner/runners/asset/tester.go index 57b763850a..d52f0bf69e 100644 --- a/internal/testrunner/runners/asset/tester.go +++ b/internal/testrunner/runners/asset/tester.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/requiredinputs" "github.com/elastic/elastic-package/internal/resources" "github.com/elastic/elastic-package/internal/testrunner" ) @@ -88,11 +89,12 @@ func (r *tester) Run(ctx context.Context) ([]testrunner.TestResult, error) { func (r *tester) resources(installedPackage bool) resources.Resources { return resources.Resources{ &resources.FleetPackage{ - PackageRoot: r.packageRoot, - Absent: !installedPackage, - Force: installedPackage, // Force re-installation, in case there are code changes in the same package version. - RepositoryRoot: r.repositoryRoot, - SchemaURLs: r.schemaURLs, + PackageRoot: r.packageRoot, + Absent: !installedPackage, + Force: installedPackage, // Force re-installation, in case there are code changes in the same package version. + RepositoryRoot: r.repositoryRoot, + SchemaURLs: r.schemaURLs, + RequiredInputsResolver: &requiredinputs.NoopRequiredInputsResolver{}, }, } } diff --git a/internal/testrunner/runners/policy/runner.go b/internal/testrunner/runners/policy/runner.go index 340577ab2b..0902decb4a 100644 --- a/internal/testrunner/runners/policy/runner.go +++ b/internal/testrunner/runners/policy/runner.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/requiredinputs" "github.com/elastic/elastic-package/internal/resources" "github.com/elastic/elastic-package/internal/testrunner" ) @@ -39,37 +40,40 @@ type runner struct { repositoryRoot *os.Root - schemaURLs fields.SchemaURLs + schemaURLs fields.SchemaURLs + requiredInputsResolver requiredinputs.Resolver } // Ensures that runner implements testrunner.TestRunner interface var _ testrunner.TestRunner = new(runner) type PolicyTestRunnerOptions struct { - KibanaClient *kibana.Client - PackageRoot string - DataStreams []string - FailOnMissingTests bool - GenerateTestResult bool - GlobalTestConfig testrunner.GlobalRunnerTestConfig - WithCoverage bool - CoverageType string - RepositoryRoot *os.Root - SchemaURLs fields.SchemaURLs + KibanaClient *kibana.Client + PackageRoot string + DataStreams []string + FailOnMissingTests bool + GenerateTestResult bool + GlobalTestConfig testrunner.GlobalRunnerTestConfig + WithCoverage bool + CoverageType string + RepositoryRoot *os.Root + SchemaURLs fields.SchemaURLs + RequiredInputsResolver requiredinputs.Resolver } func NewPolicyTestRunner(options PolicyTestRunnerOptions) *runner { runner := runner{ - packageRoot: options.PackageRoot, - kibanaClient: options.KibanaClient, - dataStreams: options.DataStreams, - failOnMissingTests: options.FailOnMissingTests, - generateTestResult: options.GenerateTestResult, - globalTestConfig: options.GlobalTestConfig, - withCoverage: options.WithCoverage, - coverageType: options.CoverageType, - repositoryRoot: options.RepositoryRoot, - schemaURLs: options.SchemaURLs, + packageRoot: options.PackageRoot, + kibanaClient: options.KibanaClient, + dataStreams: options.DataStreams, + failOnMissingTests: options.FailOnMissingTests, + generateTestResult: options.GenerateTestResult, + globalTestConfig: options.GlobalTestConfig, + withCoverage: options.WithCoverage, + coverageType: options.CoverageType, + repositoryRoot: options.RepositoryRoot, + schemaURLs: options.SchemaURLs, + requiredInputsResolver: options.RequiredInputsResolver, } runner.resourcesManager = resources.NewManager() runner.resourcesManager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: runner.kibanaClient}) @@ -169,9 +173,10 @@ func (r *runner) Type() testrunner.TestType { func (r *runner) setupSuite(ctx context.Context, manager *resources.Manager) (cleanup func(ctx context.Context) error, err error) { packageResource := resources.FleetPackage{ - PackageRoot: r.packageRoot, - RepositoryRoot: r.repositoryRoot, - SchemaURLs: r.schemaURLs, + PackageRoot: r.packageRoot, + RepositoryRoot: r.repositoryRoot, + SchemaURLs: r.schemaURLs, + RequiredInputsResolver: r.requiredInputsResolver, } setupResources := resources.Resources{ &packageResource, diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index a851c734aa..0f5360b0c3 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -19,6 +19,7 @@ import ( "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/requiredinputs" "github.com/elastic/elastic-package/internal/resources" "github.com/elastic/elastic-package/internal/servicedeployer" "github.com/elastic/elastic-package/internal/testrunner" @@ -295,11 +296,12 @@ func (r *runner) Type() testrunner.TestType { func (r *runner) resources(opts resourcesOptions) resources.Resources { return resources.Resources{ &resources.FleetPackage{ - PackageRoot: r.packageRoot, - Absent: !opts.installedPackage, - Force: opts.installedPackage, // Force re-installation, in case there are code changes in the same package version. - RepositoryRoot: r.repositoryRoot, - SchemaURLs: r.schemaURLs, + PackageRoot: r.packageRoot, + Absent: !opts.installedPackage, + Force: opts.installedPackage, // Force re-installation, in case there are code changes in the same package version. + RepositoryRoot: r.repositoryRoot, + SchemaURLs: r.schemaURLs, + RequiredInputsResolver: &requiredinputs.NoopRequiredInputsResolver{}, }, } } diff --git a/internal/testrunner/script/package.go b/internal/testrunner/script/package.go index cfcf9158e6..93c5be8bce 100644 --- a/internal/testrunner/script/package.go +++ b/internal/testrunner/script/package.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/elastic-package/internal/fields" "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/requiredinputs" "github.com/elastic/elastic-package/internal/resources" ) @@ -64,11 +65,12 @@ func addPackage(ts *testscript.TestScript, neg bool, args []string) { m := resources.NewManager() m.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: stk.kibana}) _, err = m.ApplyCtx(ctx, resources.Resources{&resources.FleetPackage{ - PackageRoot: pkgRoot, - Absent: false, - Force: true, - RepositoryRoot: root, - SchemaURLs: fields.NewSchemaURLs(fields.WithECSBaseURL(ecsBaseSchemaURL)), + PackageRoot: pkgRoot, + Absent: false, + Force: true, + RepositoryRoot: root, + SchemaURLs: fields.NewSchemaURLs(fields.WithECSBaseURL(ecsBaseSchemaURL)), + RequiredInputsResolver: &requiredinputs.NoopRequiredInputsResolver{}, }}) ts.Check(decoratedWith("installing package resources", err)) @@ -117,10 +119,11 @@ func removePackage(ts *testscript.TestScript, neg bool, args []string) { m := resources.NewManager() m.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: stk.kibana}) _, err = m.ApplyCtx(ctx, resources.Resources{&resources.FleetPackage{ - PackageRoot: pkgRoot, - Absent: true, - Force: true, - RepositoryRoot: root, // Apparently not required, but adding for safety. + PackageRoot: pkgRoot, + Absent: true, + Force: true, + RepositoryRoot: root, // Apparently not required, but adding for safety. + RequiredInputsResolver: &requiredinputs.NoopRequiredInputsResolver{}, }}) ts.Check(decoratedWith("removing package resources", err)) diff --git a/internal/testrunner/script/script.go b/internal/testrunner/script/script.go index dd524c1c44..1ef8569b51 100644 --- a/internal/testrunner/script/script.go +++ b/internal/testrunner/script/script.go @@ -34,6 +34,7 @@ import ( "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/packages/changelog" "github.com/elastic/elastic-package/internal/registry" + "github.com/elastic/elastic-package/internal/requiredinputs" "github.com/elastic/elastic-package/internal/resources" "github.com/elastic/elastic-package/internal/servicedeployer" "github.com/elastic/elastic-package/internal/stack" @@ -53,6 +54,7 @@ type Options struct { UpdateScripts bool // testscript.Params.UpdateScripts ContinueOnError bool // testscript.Params.ContinueOnError TestWork bool // testscript.Params.TestWork + } func Run(dst *[]testrunner.TestResult, w io.Writer, opt Options) error { @@ -370,9 +372,10 @@ func cleanUp(ctx context.Context, pkgRoot string, srvs map[string]servicedeploye m := resources.NewManager() m.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: stk.kibana}) _, err := m.ApplyCtx(ctx, resources.Resources{&resources.FleetPackage{ - PackageRoot: pkgRoot, - Absent: true, - Force: true, + PackageRoot: pkgRoot, + Absent: true, + Force: true, + RequiredInputsResolver: &requiredinputs.NoopRequiredInputsResolver{}, }}) if err != nil { errs = append(errs, err) diff --git a/test/manual_packages/README.md b/test/manual_packages/README.md new file mode 100644 index 0000000000..07fe651b2e --- /dev/null +++ b/test/manual_packages/README.md @@ -0,0 +1,28 @@ +# Manual Test Packages + +Packages in this directory are **not** picked up by CI build/install scripts (which glob `test/packages/*/*/`). They require manual setup to exercise. + +## required_inputs + +These packages test the `requires.input` (composable input) feature. + +- `required_inputs/test_input_pkg` — the input package that must be installed first. +- `required_inputs/with_input_package_requires` — an integration package that declares a dependency on `test_input_pkg`. + +### Manual testing workflow + +1. Start the stack and local package registry: + ```bash + elastic-package stack up -d + ``` +2. Configure `package_registry.base_url` to point at the stack's registry URL (see `scripts/test-build-install-zip.sh` lines 69–78 for the pattern). +3. Build and install in dependency order: + ```bash + elastic-package build -C test/manual_packages/required_inputs/test_input_pkg --zip + elastic-package build -C test/manual_packages/required_inputs/with_input_package_requires --zip + ``` +4. Install via the local registry, `test_input_pkg` first, then `with_input_package_requires`. + +### When composable inputs are fully supported in CI + +Move `required_inputs/` back to `test/packages/required_inputs/` so the existing install scripts regain automated coverage without requiring additional special-casing. diff --git a/test/manual_packages/required_inputs/test_input_pkg/agent/input/extra.yml.hbs b/test/manual_packages/required_inputs/test_input_pkg/agent/input/extra.yml.hbs new file mode 100644 index 0000000000..c51c9f1721 --- /dev/null +++ b/test/manual_packages/required_inputs/test_input_pkg/agent/input/extra.yml.hbs @@ -0,0 +1,2 @@ +exclude_files: + - ".gz$" \ No newline at end of file diff --git a/test/manual_packages/required_inputs/test_input_pkg/agent/input/input.yml.hbs b/test/manual_packages/required_inputs/test_input_pkg/agent/input/input.yml.hbs new file mode 100644 index 0000000000..9e9c27a8c0 --- /dev/null +++ b/test/manual_packages/required_inputs/test_input_pkg/agent/input/input.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} \ No newline at end of file diff --git a/test/manual_packages/required_inputs/test_input_pkg/changelog.yml b/test/manual_packages/required_inputs/test_input_pkg/changelog.yml new file mode 100644 index 0000000000..0f9966a2de --- /dev/null +++ b/test/manual_packages/required_inputs/test_input_pkg/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.1.0 + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/elastic-package/issues/3278 \ No newline at end of file diff --git a/test/manual_packages/required_inputs/test_input_pkg/docs/README.md b/test/manual_packages/required_inputs/test_input_pkg/docs/README.md new file mode 100644 index 0000000000..5fa7854175 --- /dev/null +++ b/test/manual_packages/required_inputs/test_input_pkg/docs/README.md @@ -0,0 +1,3 @@ +# Test Input Package + +This is a test fixture package used to verify template bundling during build. \ No newline at end of file diff --git a/test/manual_packages/required_inputs/test_input_pkg/fields/base-fields.yml b/test/manual_packages/required_inputs/test_input_pkg/fields/base-fields.yml new file mode 100644 index 0000000000..d3b0f5a163 --- /dev/null +++ b/test/manual_packages/required_inputs/test_input_pkg/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. \ No newline at end of file diff --git a/test/manual_packages/required_inputs/test_input_pkg/manifest.yml b/test/manual_packages/required_inputs/test_input_pkg/manifest.yml new file mode 100644 index 0000000000..6a83dcc4bb --- /dev/null +++ b/test/manual_packages/required_inputs/test_input_pkg/manifest.yml @@ -0,0 +1,34 @@ +format_version: 3.6.0 +name: test_input_pkg +title: Test Input Package +description: Input package used as a test fixture for template bundling. +version: 0.1.0 +type: input +categories: + - custom +conditions: + kibana: + version: "^8.0.0" + elastic: + subscription: basic +policy_templates: + - name: test_input + type: logs + title: Test Input + description: Collect test logs with a custom input template. + input: logfile + template_paths: + - input.yml.hbs + - extra.yml.hbs + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/*.log +owner: + github: elastic/integrations + type: elastic \ No newline at end of file diff --git a/test/manual_packages/required_inputs/with_input_package_requires/_dev/test/config.yml b/test/manual_packages/required_inputs/with_input_package_requires/_dev/test/config.yml new file mode 100644 index 0000000000..c4e73f3a8d --- /dev/null +++ b/test/manual_packages/required_inputs/with_input_package_requires/_dev/test/config.yml @@ -0,0 +1,3 @@ +requires: + - package: test_input_pkg + source: "../../test_input_pkg" diff --git a/test/manual_packages/required_inputs/with_input_package_requires/changelog.yml b/test/manual_packages/required_inputs/with_input_package_requires/changelog.yml new file mode 100644 index 0000000000..0f9966a2de --- /dev/null +++ b/test/manual_packages/required_inputs/with_input_package_requires/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.1.0 + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/elastic-package/issues/3278 \ No newline at end of file diff --git a/test/manual_packages/required_inputs/with_input_package_requires/data_stream/test_logs/agent/stream/stream.yml.hbs b/test/manual_packages/required_inputs/with_input_package_requires/data_stream/test_logs/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..9e9c27a8c0 --- /dev/null +++ b/test/manual_packages/required_inputs/with_input_package_requires/data_stream/test_logs/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} \ No newline at end of file diff --git a/test/manual_packages/required_inputs/with_input_package_requires/data_stream/test_logs/fields/base-fields.yml b/test/manual_packages/required_inputs/with_input_package_requires/data_stream/test_logs/fields/base-fields.yml new file mode 100644 index 0000000000..d3b0f5a163 --- /dev/null +++ b/test/manual_packages/required_inputs/with_input_package_requires/data_stream/test_logs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. \ No newline at end of file diff --git a/test/manual_packages/required_inputs/with_input_package_requires/data_stream/test_logs/manifest.yml b/test/manual_packages/required_inputs/with_input_package_requires/data_stream/test_logs/manifest.yml new file mode 100644 index 0000000000..4e198906eb --- /dev/null +++ b/test/manual_packages/required_inputs/with_input_package_requires/data_stream/test_logs/manifest.yml @@ -0,0 +1,19 @@ +title: Test Logs +type: logs +streams: + - package: test_input_pkg + title: Test Logs from Input Package + description: Collect test logs using the referenced input package. + - input: logs + title: Test Logs + description: Collect test logs using the logs input. + template_path: stream.yml.hbs + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/test/*.log \ No newline at end of file diff --git a/test/manual_packages/required_inputs/with_input_package_requires/docs/README.md b/test/manual_packages/required_inputs/with_input_package_requires/docs/README.md new file mode 100644 index 0000000000..22de7c28e4 --- /dev/null +++ b/test/manual_packages/required_inputs/with_input_package_requires/docs/README.md @@ -0,0 +1,4 @@ +# Integration With Required Input Package + +This is a test fixture integration package that demonstrates template bundling +from a required input package. \ No newline at end of file diff --git a/test/manual_packages/required_inputs/with_input_package_requires/manifest.yml b/test/manual_packages/required_inputs/with_input_package_requires/manifest.yml new file mode 100644 index 0000000000..7a6ad33229 --- /dev/null +++ b/test/manual_packages/required_inputs/with_input_package_requires/manifest.yml @@ -0,0 +1,34 @@ +format_version: 3.6.0 +name: with_input_package_requires +title: Integration With Required Input Package +description: >- + Integration package that requires an input package, used to test template bundling. +version: 0.1.0 +type: integration +categories: + - custom +conditions: + kibana: + version: "^8.0.0" + elastic: + subscription: basic +requires: + input: + - package: test_input_pkg + version: "0.1.0" +policy_templates: + - name: test_logs + title: Test logs + description: Collect test logs + data_streams: + - test_logs + inputs: + - package: test_input_pkg + title: Collect test logs via input package + description: Use the test input package to collect logs + - type: logs + title: Collect test logs via logs input + description: Use the logs input to collect logs +owner: + github: elastic/integrations + type: elastic \ No newline at end of file