diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index f3de81d4eac..e4f09c24735 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -118,6 +118,36 @@ steps: imagePrefix: "core-ubuntu-2204-aarch64" diskSizeGb: 200 + - label: "Packaging: Containers linux/amd64 FIPS" + key: packaging-containers-x86-64-fips + env: + PACKAGES: "docker" + PLATFORMS: "linux/amd64" + FIPS: "true" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + diskSizeGb: 200 + + - label: "Packaging: Containers linux/arm64 FIPS" + key: packaging-containers-arm64-fips + env: + PACKAGES: "docker" + PLATFORMS: "linux/arm64" + FIPS: "true" + command: | + .buildkite/scripts/steps/integration-package.sh + artifact_paths: + - build/distributions/** + agents: + provider: "aws" + instanceType: "c6g.4xlarge" + imagePrefix: "core-ubuntu-2204-aarch64" + diskSizeGb: 200 + - label: "Serverless integration test" key: "serverless-integration-tests" depends_on: diff --git a/.buildkite/pipeline.elastic-agent-binary-dra.yml b/.buildkite/pipeline.elastic-agent-binary-dra.yml index 02bbffe3d07..ea3cfd762e4 100644 --- a/.buildkite/pipeline.elastic-agent-binary-dra.yml +++ b/.buildkite/pipeline.elastic-agent-binary-dra.yml @@ -25,7 +25,21 @@ steps: env: DRA_WORKFLOW: "snapshot" PLATFORMS: "linux/amd64 windows/amd64 darwin/amd64" - + + - label: ":package: linux/amd64 FIPS Elastic-Agent Core Snapshot" + commands: + - .buildkite/scripts/steps/build-agent-core.sh + key: "build-dra-snapshot-x86-fips" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + DRA_WORKFLOW: "snapshot" + PLATFORMS: "linux/amd64" + FIPS: "true" + - label: ":package: linux/arm64 darwin/arm64 Elastic-Agent Core Snapshot" commands: - .buildkite/scripts/steps/build-agent-core.sh @@ -40,6 +54,20 @@ steps: DRA_WORKFLOW: "snapshot" PLATFORMS: "linux/arm64 darwin/arm64" + - label: ":package: linux/arm64 FIPS Elastic-Agent Core Snapshot" + commands: + - .buildkite/scripts/steps/build-agent-core.sh + key: "build-dra-snapshot-arm-fips" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "aws" + instanceType: "c6g.4xlarge" + imagePrefix: "core-ubuntu-2204-aarch64" + env: + DRA_WORKFLOW: "snapshot" + PLATFORMS: "linux/arm64" + FIPS: "true" - wait - label: ":hammer: DRA Publish Elastic-Agent Core Snapshot" @@ -86,6 +114,21 @@ steps: DRA_WORKFLOW: "staging" PLATFORMS: "linux/amd64 windows/amd64 darwin/amd64" + - label: ":package: linux/amd64 FIPS Elastic-Agent Core staging" + commands: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/steps/build-agent-core.sh + key: "build-dra-staging-x86-fips" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + DRA_WORKFLOW: "staging" + PLATFORMS: "linux/amd64" + FIPS: "true" + - label: ":package: linux/arm64 darwin/arm64 Elastic-Agent Core staging" commands: | source .buildkite/scripts/version_qualifier.sh @@ -101,6 +144,22 @@ steps: DRA_WORKFLOW: "dra-core-staging" PLATFORMS: "linux/arm64 darwin/arm64" + - label: ":package: linux/arm64 FIPS Elastic-Agent Core staging" + commands: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/steps/build-agent-core.sh + key: "build-dra-staging-arm-fips" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "aws" + instanceType: "c6g.4xlarge" + imagePrefix: "core-ubuntu-2204-aarch64" + env: + DRA_WORKFLOW: "dra-core-staging" + PLATFORMS: "linux/arm64" + FIPS: "true" + - wait - label: ":hammer: DRA Publish Elastic-Agent Core staging" diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 92da55273b1..c5d879a6510 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -18,6 +18,8 @@ import ( "github.com/magefile/mage/sh" "golang.org/x/text/cases" "golang.org/x/text/language" + + "github.com/elastic/elastic-agent/dev-tools/packaging" ) // BuildArgs are the arguments used for the "build" target and they define how @@ -73,6 +75,7 @@ func DefaultBuildArgs() BuildArgs { args := BuildArgs{ Name: BeatName, CGO: build.Default.CgoEnabled, + Env: map[string]string{}, Vars: map[string]string{ elasticAgentModulePath + "/version.buildTime": "{{ date }}", elasticAgentModulePath + "/version.commit": "{{ commit }}", @@ -88,8 +91,16 @@ func DefaultBuildArgs() BuildArgs { } if FIPSBuild { - args.ExtraFlags = append(args.ExtraFlags, "-tags=requirefips") - args.CGO = true + + fipsConfig := packaging.Settings().FIPS + + for _, tag := range fipsConfig.Compile.Tags { + args.ExtraFlags = append(args.ExtraFlags, "-tags="+tag) + } + args.CGO = args.CGO || fipsConfig.Compile.CGO + for varName, value := range fipsConfig.Compile.Env { + args.Env[varName] = value + } } if DevBuild { @@ -191,11 +202,6 @@ func Build(params BuildArgs) error { cgoEnabled = "1" } - if FIPSBuild { - cgoEnabled = "1" - env["GOEXPERIMENT"] = "systemcrypto" - } - env["CGO_ENABLED"] = cgoEnabled // Spec diff --git a/dev-tools/mage/checksums.go b/dev-tools/mage/checksums.go index b12a4e3ec3e..d885d70f7ca 100644 --- a/dev-tools/mage/checksums.go +++ b/dev-tools/mage/checksums.go @@ -9,12 +9,12 @@ import ( "log" "os" "path/filepath" - "strings" "github.com/magefile/mage/mg" "github.com/otiai10/copy" "github.com/elastic/elastic-agent/dev-tools/mage/manifest" + "github.com/elastic/elastic-agent/dev-tools/packaging" ) const ComponentSpecFileSuffix = ".spec.yml" @@ -40,23 +40,53 @@ func CopyComponentSpecs(componentName, versionedDropPath string) (string, error) return GetSHA512Hash(targetPath) } -// This is a helper function for flattenDependencies that's used when not packaging from a manifest -func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string, packageVersion string) map[string]string { - globExpr := filepath.Join(versionedFlatPath, fmt.Sprintf("*%s*", packageVersion)) - if mg.Verbose() { - log.Printf("Finding files to copy with %s", globExpr) - } - files, err := filepath.Glob(globExpr) - if err != nil { - panic(err) - } - if mg.Verbose() { - log.Printf("Validating checksums for %+v", files) - log.Printf("--- Copying into %s: %v", versionedDropPath, files) - } - +// ChecksumsWithoutManifest is a helper function for flattenDependencies that's used when not packaging from a manifest. +// This function will iterate over the dependencies, resolve *exactly* the package name for each dependency and platform using the passed +// dependenciesVersion, and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath +// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform +// that will be used to compose the package content) +// ChecksumsWithoutManifest will accumulate the checksums of each component spec that is copied, and return it to the caller. +func ChecksumsWithoutManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, dependencies []packaging.BinarySpec) map[string]string { checksums := make(map[string]string) - for _, f := range files { + + for _, dep := range dependencies { + + if dep.PythonWheel { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", dep.ProjectName, dep.BinaryName) + } + continue + } + + if !dep.SupportsPlatform(platform) { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", dep.ProjectName, dep.BinaryName, platform) + } + continue + } + + atLeastOnePackageTypeSelected := false + for _, pkgType := range dep.PackageTypes { + if IsPackageTypeSelected(PackageType(pkgType)) { + atLeastOnePackageTypeSelected = true + break + } + } + + if !atLeastOnePackageTypeSelected { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s supported package types %v do not overlap selected package types %v, skipping", dep.ProjectName, dep.BinaryName, dep.PackageTypes, SelectedPackageTypes) + } + continue + } + + srcDir := filepath.Join(versionedFlatPath, dep.GetRootDir(dependenciesVersion, platform)) + + if mg.Verbose() { + log.Printf("Validating checksums for %+v", dep.BinaryName) + log.Printf("--- Copying into %s: %v", versionedDropPath, srcDir) + } + options := copy.Options{ OnSymlink: func(_ string) copy.SymlinkAction { return copy.Shallow @@ -64,222 +94,99 @@ func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string Sync: true, } if mg.Verbose() { - log.Printf("> prepare to copy %s into %s ", f, versionedDropPath) + log.Printf("> prepare to copy %s into %s ", srcDir, versionedDropPath) } - err = copy.Copy(f, versionedDropPath, options) + err := copy.Copy(srcDir, versionedDropPath, options) if err != nil { - panic(err) + panic(fmt.Errorf("copying dependency %s files from %q to %q: %w", dep.BinaryName, srcDir, versionedDropPath, err)) } // copy spec file for match - specName := filepath.Base(f) - idx := strings.Index(specName, "-"+packageVersion) - if idx != -1 { - specName = specName[:idx] - } if mg.Verbose() { - log.Printf(">>>> Looking to copy spec file: [%s]", specName) + log.Printf(">>>> Looking to copy spec file: [%s]", dep.BinaryName) } - checksum, err := CopyComponentSpecs(specName, versionedDropPath) + checksum, err := CopyComponentSpecs(dep.BinaryName, versionedDropPath) if err != nil { panic(err) } - checksums[specName+ComponentSpecFileSuffix] = checksum + checksums[dep.BinaryName+ComponentSpecFileSuffix] = checksum } return checksums } -// This is a helper function for flattenDependencies that's used when building from a manifest -func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build) map[string]string { +// ChecksumsWithManifest is a helper function for flattenDependencies that's used when building from a manifest. +// This function will iterate over the dependencies, resolve the package name for each dependency and platform using the manifest, +// (there may be some variability there in case the manifest does not include an exact match for the expected filename), +// and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath +// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform +// that will be used to compose the package content) +// ChecksumsWithManifest will accumulate the checksums of each component spec that is copied, and return it to the caller. +func ChecksumsWithManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build, dependencies []packaging.BinarySpec) map[string]string { checksums := make(map[string]string) if manifestResponse == nil { return checksums } - // Iterate over the component projects in the manifest - projects := manifestResponse.Projects - for componentName := range projects { - // Iterate over the individual package files within each component project - for pkgName := range projects[componentName].Packages { - // Only care about packages that match the required package constraint (os/arch) - if strings.Contains(pkgName, requiredPackage) { - // Iterate over the external binaries that we care about for packaging agent - for _, spec := range manifest.ExpectedBinaries { - // If the individual package doesn't match the expected prefix, then continue - // FIXME temporarily skip fips packages until elastic-agent FIPS is in place - if !strings.HasPrefix(pkgName, spec.BinaryName) || strings.Contains(pkgName, "-fips-") { - if mg.Verbose() { - log.Printf(">>>>>>> Package [%s] skipped", pkgName) - } - continue - } - - if mg.Verbose() { - log.Printf(">>>>>>> Package [%s] matches requiredPackage [%s]", pkgName, requiredPackage) - } - - // Get the version from the component based on the version in the package name - // This is useful in the case where it's an Independent Agent Release, where - // the opted-in projects will be one patch version ahead of the rest of the - // opted-out/previously-released projects - componentVersion := getComponentVersion(componentName, requiredPackage, projects[componentName]) - if mg.Verbose() { - log.Printf(">>>>>>> Component [%s]/[%s] version is [%s]", componentName, requiredPackage, componentVersion) - } - - // Combine the package name w/ the versioned flat path - fullPath := filepath.Join(versionedFlatPath, pkgName) - - // Eliminate the file extensions to get the proper directory - // name that we need to copy - var dirToCopy string - if strings.HasSuffix(fullPath, ".tar.gz") { - dirToCopy = fullPath[:strings.LastIndex(fullPath, ".tar.gz")] - } else if strings.HasSuffix(fullPath, ".zip") { - dirToCopy = fullPath[:strings.LastIndex(fullPath, ".zip")] - } else { - dirToCopy = fullPath - } - if mg.Verbose() { - log.Printf(">>>>>>> Calculated directory to copy: [%s]", dirToCopy) - } - - // cloud-defend path exception - // When untarred, cloud defend untars to: - // cloud-defend-8.14.0-arm64 - // but the manifest (and most of this code) expects to be the same as - // the name in the manifest, which is: - // cloud-defend-8.14.0-linux-x86_64 - // So we have to do a bit of a transformation here - if strings.Contains(dirToCopy, "cloud-defend") { - if strings.Contains(dirToCopy, "x86_64") { - dirToCopy = fixCloudDefendDirPath(dirToCopy, componentVersion, "x86_64", "amd64") - } - if strings.Contains(dirToCopy, "arm64") { - // Not actually replacing the arch, but removing the "linux" - dirToCopy = fixCloudDefendDirPath(dirToCopy, componentVersion, "arm64", "arm64") - } - if mg.Verbose() { - log.Printf(">>>>>>> Adjusted cloud-defend directory to copy: [%s]", dirToCopy) - } - } - - // Set copy options - options := copy.Options{ - OnSymlink: func(_ string) copy.SymlinkAction { - return copy.Shallow - }, - Sync: true, - } - if mg.Verbose() { - log.Printf("> prepare to copy %s into %s ", dirToCopy, versionedDropPath) - } - - // Do the copy - err := copy.Copy(dirToCopy, versionedDropPath, options) - if err != nil { - panic(err) - } - - // copy spec file for match - specName := filepath.Base(dirToCopy) - idx := strings.Index(specName, "-"+componentVersion) - if idx != -1 { - specName = specName[:idx] - } - if mg.Verbose() { - log.Printf(">>>> Looking to copy spec file: [%s]", specName) - } - - checksum, err := CopyComponentSpecs(specName, versionedDropPath) - if err != nil { - panic(err) - } - - checksums[specName+ComponentSpecFileSuffix] = checksum - } + // Iterate over the external binaries that we care about for packaging agent + for _, spec := range dependencies { + + if spec.PythonWheel { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", spec.ProjectName, spec.BinaryName) } + continue } - } - return checksums -} + if !spec.SupportsPlatform(platform) { + log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", spec.ProjectName, spec.BinaryName, platform) + continue + } -// This function is used when building with a Manifest. In that manifest, it's possible -// for projects in an Independent Agent Release to have different versions since the opted-in -// ones will be one patch version higher than the opted-out/previously released projects. -// This function tries to find the versions from the package name -func getComponentVersion(componentName string, requiredPackage string, componentProject manifest.Project) string { - var componentVersion string - var foundIt bool - // Iterate over all the packages in the component project - for pkgName := range componentProject.Packages { - // Only care about the external binaries that we want to package - for _, spec := range manifest.ExpectedBinaries { - // If the given component name doesn't match the external binary component, skip - // FIXME temporarily skip fips packages until elastic-agent FIPS is in place - if componentName != spec.ProjectName || strings.Contains(pkgName, "-fips-") { - continue + manifestPackage, err := manifest.ResolveManifestPackage(manifestResponse.Projects[spec.ProjectName], spec, dependenciesVersion, platform) + if err != nil { + if mg.Verbose() { + log.Printf(">>>>>>> Error resolving package for [%s/%s]", spec.BinaryName, platform) } + continue + } - // Split the package name on the binary name prefix plus a dash - firstSplit := strings.Split(pkgName, spec.BinaryName+"-") - if len(firstSplit) < 2 { - continue - } + rootDir := spec.GetRootDir(manifestPackage.ActualVersion, platform) - // Get the second part of the first split - secondHalf := firstSplit[1] - if len(secondHalf) < 2 { - continue - } + // Combine the package name w/ the versioned flat path + fullPath := filepath.Join(versionedFlatPath, rootDir) - // Make sure the second half matches the required package - if strings.Contains(secondHalf, requiredPackage) { - // ignore packages with names where this splitting doesn't results in proper version - if strings.Contains(secondHalf, "docker-image") { - continue - } - if strings.Contains(secondHalf, "oss-") { - continue - } - - // The component version should be the first entry after splitting w/ the requiredPackage - componentVersion = strings.Split(secondHalf, "-"+requiredPackage)[0] - foundIt = true - // break out of inner loop - break - } - } - if foundIt { - // break out of outer loop - break + if mg.Verbose() { + log.Printf(">>>>>>> Calculated directory to copy: [%s]", fullPath) } - } - if componentVersion == "" { - errMsg := fmt.Sprintf("Unable to determine component version for [%s]", componentName) - panic(errMsg) - } + // Set copy options + options := copy.Options{ + OnSymlink: func(_ string) copy.SymlinkAction { + return copy.Shallow + }, + Sync: true, + } + if mg.Verbose() { + log.Printf("> prepare to copy %s into %s ", fullPath, versionedDropPath) + } - return componentVersion -} + // Do the copy + err = copy.Copy(fullPath, versionedDropPath, options) + if err != nil { + panic(err) + } -// This is a helper function for the cloud-defend package. -// When it is untarred, it does not have the same dirname as the package name. -// This adjusts for that and returns the actual path on disk for cloud-defend -func fixCloudDefendDirPath(dirPath string, componentVersion string, expectedArch string, actualArch string) string { - fixedDirPath := dirPath + checksum, err := CopyComponentSpecs(spec.BinaryName, versionedDropPath) + if err != nil { + panic(err) + } - cloudDefendExpectedDirName := fmt.Sprintf("cloud-defend-%s-linux-%s", componentVersion, expectedArch) - cloudDefendActualDirName := fmt.Sprintf("cloud-defend-%s-%s", componentVersion, actualArch) - if strings.Contains(fixedDirPath, cloudDefendExpectedDirName) { - fixedDirPath = strings.ReplaceAll(fixedDirPath, cloudDefendExpectedDirName, cloudDefendActualDirName) + checksums[spec.BinaryName+ComponentSpecFileSuffix] = checksum } - return fixedDirPath + return checksums } diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index c7340490d41..8c4e70d150e 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -30,11 +30,11 @@ const defaultCrossBuildTarget = "golangCrossBuild" var Platforms = BuildPlatforms.Defaults() // SelectedPackageTypes is the list of package types. If empty, all packages types -// are considered to be selected (see isPackageTypeSelected). +// are considered to be selected (see IsPackageTypeSelected). var SelectedPackageTypes []PackageType // SelectedDockerVariants is the list of docker variants. If empty, all docker variants -// are considered to be selected (see isDockerVariantSelected). +// are considered to be selected (see IsDockerVariantSelected). var SelectedDockerVariants []DockerVariant func init() { diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 6a0cd8609ac..a3290d69a1a 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -182,9 +182,7 @@ func (b *dockerBuilder) dockerBuild() (string, error) { if b.Snapshot { tag = tag + "-SNAPSHOT" } - if b.FIPS { - tag = tag + "-fips" - } + if repository := b.ExtraVars["repository"]; repository != "" { tag = fmt.Sprintf("%s/%s", repository, tag) } diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go index 1db9c21dbe1..7c33f08e8b3 100644 --- a/dev-tools/mage/manifest/manifest.go +++ b/dev-tools/mage/manifest/manifest.go @@ -13,13 +13,14 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" "time" "github.com/magefile/mage/mg" "golang.org/x/sync/errgroup" - "github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon" + "github.com/elastic/elastic-agent/dev-tools/packaging" "github.com/elastic/elastic-agent/pkg/version" ) @@ -94,73 +95,6 @@ var PlatformPackages = map[string]string{ "windows/amd64": "windows-x86_64.zip", } -// ExpectedBinaries is a map of binaries agent needs to their project in the unified-release manager. -// The project names are those used in the "projects" list in the unified release manifest. -// See the sample manifests in the testdata directory. -var ExpectedBinaries = []BinarySpec{ - {BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "cloud-defend", ProjectName: "cloud-defend", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: []pkgcommon.PackageType{pkgcommon.Docker}}, - {BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms, PackageTypes: []pkgcommon.PackageType{pkgcommon.RPM, pkgcommon.Deb, pkgcommon.Zip, pkgcommon.TarGz}}, - {BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, -} - -type BinarySpec struct { - BinaryName string - ProjectName string - Platforms []Platform - PythonWheel bool - PackageTypes []pkgcommon.PackageType -} - -func (proj BinarySpec) SupportsPlatform(platform string) bool { - for _, p := range proj.Platforms { - if p.Platform() == platform { - return true - } - } - return false -} - -func (proj BinarySpec) SupportsPackageType(pkgType pkgcommon.PackageType) bool { - for _, p := range proj.PackageTypes { - if p == pkgType { - return true - } - } - return false -} - -func (proj BinarySpec) GetPackageName(version string, platform string) string { - if proj.PythonWheel { - return fmt.Sprintf("%s-%s.zip", proj.BinaryName, version) - } - return fmt.Sprintf("%s-%s-%s", proj.BinaryName, version, PlatformPackages[platform]) -} - -type Platform struct { - OS string - Arch string -} - -// Converts to the format expected on the mage command line "linux", "x86_64" = "linux/amd64" -func (p Platform) Platform() string { - if p.Arch == "x86_64" { - p.Arch = "amd64" - } - if p.Arch == "aarch64" { - p.Arch = "arm64" - } - return p.OS + "/" + p.Arch -} - -var AllPlatforms = []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}, {"darwin", "aarch64"}} - // DownloadManifest is going to download the given manifest file and return the ManifestResponse func DownloadManifest(ctx context.Context, manifest string) (Build, error) { manifestUrl, urlError := url.Parse(manifest) @@ -192,7 +126,7 @@ func DownloadManifest(ctx context.Context, manifest string) (Build, error) { // DownloadComponents is going to download a set of components from the given manifest into the destination // dropPath folder in order to later use that folder for packaging -func DownloadComponents(ctx context.Context, manifest string, platforms []string, dropPath string) error { +func DownloadComponents(ctx context.Context, expectedBinaries []packaging.BinarySpec, manifest string, platforms []string, dropPath string) error { manifestResponse, err := DownloadManifest(ctx, manifest) if err != nil { return fmt.Errorf("failed to download remote manifest file %w", err) @@ -211,7 +145,7 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string errGrp, downloadsCtx := errgroup.WithContext(ctx) // for project, pkgs := range expectedProjectPkgs() { - for _, spec := range ExpectedBinaries { + for _, spec := range expectedBinaries { for _, platform := range platforms { targetPath := filepath.Join(dropPath) err := os.MkdirAll(targetPath, 0755) @@ -225,12 +159,12 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string continue } - pkgURL, err := resolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform) + resolvedPackage, err := ResolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform) if err != nil { return err } - for _, p := range pkgURL { + for _, p := range resolvedPackage.URLs { log.Printf(">>>>>>>>> Downloading [%s] [%s] ", spec.BinaryName, p) pkgFilename := path.Base(p) downloadTarget := filepath.Join(targetPath, pkgFilename) @@ -252,76 +186,121 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string return nil } -func resolveManifestPackage(project Project, spec BinarySpec, version string, platform string) ([]string, error) { - var val Package - var ok bool +type ResolvedPackage struct { + Name string + ActualVersion string + URLs []string +} + +func ResolveManifestPackage(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) { // Try the normal/easy case first - packageName := spec.GetPackageName(version, platform) - val, ok = project.Packages[packageName] - if !ok { - // If we didn't find it, it may be an Independent Agent Release, where - // the opted-in projects will have a patch version one higher than - // the rest of the projects, so we need to seek that out + packageName := spec.GetPackageName(dependencyVersion, platform) + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Got packagename [%s], looking for exact match", packageName) + } + + if exactMatch, ok := project.Packages[packageName]; ok { + // We found the exact filename we are looking for if mg.Verbose() { - log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", spec.BinaryName, PlatformPackages[platform]) + log.Printf(">>>>>>>>>>> Found exact match packageName for [%s, %s]: %s", project.Branch, project.CommitHash, exactMatch) } - var foundIt bool - for pkgName := range project.Packages { - if strings.HasPrefix(pkgName, spec.BinaryName) { - firstSplit := strings.Split(pkgName, spec.BinaryName+"-") - if len(firstSplit) < 2 { - continue - } + return &ResolvedPackage{ + Name: packageName, + ActualVersion: dependencyVersion, + URLs: []string{exactMatch.URL, exactMatch.ShaURL, exactMatch.AscURL}, + }, nil + } - secondHalf := firstSplit[1] - // Make sure we're finding one w/ the same required package type - if strings.Contains(secondHalf, PlatformPackages[platform]) { - - // Split again after the version with the required package string - secondSplit := strings.Split(secondHalf, "-"+PlatformPackages[platform]) - if len(secondSplit) < 2 { - continue - } - - // The first element after the split should normally be the version - pkgVersion := secondSplit[0] - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Using derived version for package [%s]: %s ", pkgName, pkgVersion) - } - - // Create a project/package key with the package, derived version, and required package - foundPkgKey := fmt.Sprintf("%s-%s-%s", spec.BinaryName, pkgVersion, PlatformPackages[platform]) - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Looking for project package key: [%s]", foundPkgKey) - } - - // Get the package value, if it exists - val, ok = project.Packages[foundPkgKey] - if !ok { - continue - } - - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Found package key [%s]", foundPkgKey) - } - - foundIt = true - } - } - } + // If we didn't find it, it may be an Independent Agent Release, where + // the opted-in projects will have a patch version one higher than + // the rest of the projects, so we "relax" the version constraint + return resolveManifestPackageUsingRelaxedVersion(project, spec, dependencyVersion, platform) +} - if !foundIt { - return nil, fmt.Errorf("package [%s] not found in project manifest at %s", packageName, project.ExternalArtifactsManifestURL) - } +func resolveManifestPackageUsingRelaxedVersion(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) { + // start with the rendered package name + packageName := spec.GetPackageName(dependencyVersion, platform) + + // Find the original version in the rendered filename + versionIndex := strings.Index(packageName, dependencyVersion) + if versionIndex == -1 { + return nil, fmt.Errorf("no exact match and filename %q does not seem to contain dependencyVersion %q to try a fallback", packageName, dependencyVersion) + } + + // obtain a regexp from the exact version string that allows for some flexibility on patch version, prerelease and build metadata tokens + relaxedVersion, err := relaxVersion(dependencyVersion) + if err != nil { + return nil, fmt.Errorf("relaxing dependencyVersion %q: %w", dependencyVersion, err) } if mg.Verbose() { - log.Printf(">>>>>>>>>>> Project branch/commit [%s, %s]", project.Branch, project.CommitHash) + log.Printf(">>>>>>>>>>> Couldn't find exact match, relaxing agent dependencyVersion to %s", relaxedVersion) } - return []string{val.URL, val.ShaURL, val.AscURL}, nil + // locate the original version in the filename and substitute the relaxed version regexp, quoting everything around that + relaxedPackageName := regexp.QuoteMeta(packageName[:versionIndex]) + relaxedPackageName += `(?P` + relaxedVersion + `)` + relaxedPackageName += regexp.QuoteMeta(packageName[versionIndex+len(dependencyVersion):]) + + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Attempting to match a filename with %s", relaxedPackageName) + } + + relaxedPackageNameRegexp, err := regexp.Compile(relaxedPackageName) + if err != nil { + return nil, fmt.Errorf("compiling relaxed package name regex %q: %w", relaxedPackageName, err) + } + + for pkgName, pkg := range project.Packages { + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Evaluating filename %s", pkgName) + } + if submatches := relaxedPackageNameRegexp.FindStringSubmatch(pkgName); len(submatches) > 0 { + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Found matching packageName for [%s, %s]: %s", project.Branch, project.CommitHash, pkgName) + } + return &ResolvedPackage{ + Name: pkgName, + ActualVersion: submatches[1], + URLs: []string{pkg.URL, pkg.ShaURL, pkg.AscURL}, + }, nil + } + } + + return nil, fmt.Errorf("package [%s] not found in project manifest at %s using relaxed version %q", packageName, project.ExternalArtifactsManifestURL, relaxedPackageName) +} + +// versionRegexp is taken from https://semver.org/ (see the FAQ section/Is there a suggested regular expression (RegEx) to check a SemVer string?) +const versionRegexp = `^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?:[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$` +const anyPatchVersionRegexp = `(?:0|[1-9]\d*)` + +var versionRegExp = regexp.MustCompile(versionRegexp) + +func relaxVersion(version string) (string, error) { + matchIndices := versionRegExp.FindSubmatchIndex([]byte(version)) + // Matches index pairs are (0,1) for the whole regexp and (2,3) for the patch group + // check that we have matched correctly + if len(matchIndices) < 4 { + return "", fmt.Errorf("failed to match regexp for version [%s]", version) + } + + // take the starting index of the patch version + patchStartIndex := matchIndices[2] + // copy everything before the patch version escaping the regexp + relaxedVersion := regexp.QuoteMeta(version[:patchStartIndex]) + // add the patch regexp + relaxedVersion += anyPatchVersionRegexp + // check if there's more characters after the patch version + remainderIndex := matchIndices[3] + if remainderIndex < len(version) { + // This is a looser regexp that allows anything beyond the major version to change (while still enforcing a valid patch version though) + // see TestResolveManifestPackage/Independent_Agent_Staging_8.14_apm-server and TestResolveManifestPackage/Independent_Agent_Staging_8.14_endpoint-dev + // Be more relaxed and allow for any character sequence after this + relaxedVersion += `.*` + } + return relaxedVersion, nil } func DownloadPackage(ctx context.Context, downloadUrl string, target string) error { @@ -331,12 +310,12 @@ func DownloadPackage(ctx context.Context, downloadUrl string, target string) err } valid := false for _, manifestHost := range AllowedManifestHosts { - if manifestHost == parsedURL.Host { + if manifestHost == parsedURL.Hostname() { valid = true } } if !valid { - log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Host, AllowedManifestHosts) + log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Hostname(), AllowedManifestHosts) return errorNotAllowedManifestURL } cleanUrl := fmt.Sprintf("https://%s%s", parsedURL.Host, parsedURL.Path) diff --git a/dev-tools/mage/manifest/manifest_test.go b/dev-tools/mage/manifest/manifest_test.go index b975149aa27..76a0d53cd2e 100644 --- a/dev-tools/mage/manifest/manifest_test.go +++ b/dev-tools/mage/manifest/manifest_test.go @@ -7,12 +7,15 @@ package manifest import ( _ "embed" "encoding/json" + "fmt" "log" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent/dev-tools/packaging" ) var ( @@ -135,7 +138,7 @@ func TestResolveManifestPackage(t *testing.T) { projects := manifestJson.Projects // Verify the component name is in the list of expected packages. - spec, ok := findBinarySpec(tc.binary) + spec, ok := findBinarySpec(t, tc.binary) assert.True(t, ok) if !spec.SupportsPlatform(tc.platform) { @@ -143,22 +146,80 @@ func TestResolveManifestPackage(t *testing.T) { return } - urlList, err := resolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform) + resolvedPackage, err := ResolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform) require.NoError(t, err) + require.NotNil(t, resolvedPackage) - assert.Len(t, urlList, 3) - for _, url := range urlList { + assert.Len(t, resolvedPackage.URLs, 3) + for _, url := range resolvedPackage.URLs { assert.Contains(t, tc.expectedUrlList, url) } }) } } -func findBinarySpec(name string) (BinarySpec, bool) { - for _, spec := range ExpectedBinaries { +func findBinarySpec(t *testing.T, name string) (packaging.BinarySpec, bool) { + components, err := packaging.Components() + require.NoError(t, err, "error loading components from packages.yml") + + for _, spec := range components { if spec.BinaryName == name { return spec, true } } - return BinarySpec{}, false + return packaging.BinarySpec{}, false +} + +func TestRelaxVersion(t *testing.T) { + type args struct { + version string + } + tests := []struct { + name string + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "major-minor-patch", + args: args{ + version: "1.2.3", + }, + want: `1\.2\.(?:0|[1-9]\d*)`, + wantErr: assert.NoError, + }, + { + name: "major-minor-patch-snapshot", + args: args{ + version: "1.2.3-SNAPSHOT", + }, + want: `1\.2\.(?:0|[1-9]\d*).*`, + wantErr: assert.NoError, + }, + { + name: "major-minor-patch-snapshot-buildmeta", + args: args{ + version: "1.2.3-SNAPSHOT+build20250328112233", + }, + want: `1\.2\.(?:0|[1-9]\d*).*`, + wantErr: assert.NoError, + }, + { + name: "not a semver", + args: args{ + version: "foobar", + }, + want: "", + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := relaxVersion(tt.args.version) + if !tt.wantErr(t, err, fmt.Sprintf("relaxVersion(%v)", tt.args.version)) { + return + } + assert.Equalf(t, tt.want, got, "relaxVersion(%v)", tt.args.version) + }) + } } diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index 6ec09f2e598..2103305e48d 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -34,6 +34,15 @@ func Package() error { // platforms := updateWithDarwinUniversal(Platforms) platforms := Platforms + if mg.Verbose() { + debugSelectedPackageSpecsWithPlatform := make([]string, 0, len(Packages)) + for _, p := range Packages { + debugSelectedPackageSpecsWithPlatform = append(debugSelectedPackageSpecsWithPlatform, fmt.Sprintf("spec %s on %s/%s", p.Spec.Name, p.OS, p.Arch)) + } + + log.Printf("Packaging for platforms %v, packages %v", platforms, debugSelectedPackageSpecsWithPlatform) + } + tasks := make(map[string][]interface{}) for _, target := range platforms { for _, pkg := range Packages { @@ -41,13 +50,19 @@ func Package() error { continue } + // Checks if this package is compatible with the FIPS settings + if pkg.Spec.FIPS != FIPSBuild { + log.Printf("Skipping %s/%s package type because FIPS flag doesn't match [pkg=%v, build=%v]", pkg.Spec.Name, pkg.OS, pkg.Spec.FIPS, FIPSBuild) + continue + } + for _, pkgType := range pkg.Types { - if !isPackageTypeSelected(pkgType) { + if !IsPackageTypeSelected(pkgType) { log.Printf("Skipping %s package type because it is not selected", pkgType) continue } - if pkgType == Docker && !isDockerVariantSelected(pkg.Spec.DockerVariant) { + if pkgType == Docker && !IsDockerVariantSelected(pkg.Spec.DockerVariant) { log.Printf("Skipping %s docker variant type because it is not selected", pkg.Spec.DockerVariant) continue } @@ -80,7 +95,6 @@ func Package() error { spec.OS = target.GOOS() spec.Arch = packageArch spec.Snapshot = Snapshot - spec.FIPS = FIPSBuild spec.evalContext = map[string]interface{}{ "GOOS": target.GOOS(), "GOARCH": target.GOARCH(), @@ -100,6 +114,10 @@ func Package() error { spec = spec.Evaluate() + if mg.Verbose() { + log.Printf("Adding task for packaging %s on %s/%s", spec.Name, target.GOOS(), target.Arch()) + } + tasks[target.GOOS()+"-"+target.Arch()] = append(tasks[target.GOOS()+"-"+target.Arch()], packageBuilder{target, spec, pkgType}.Build) } } @@ -112,9 +130,9 @@ func Package() error { return nil } -// isPackageTypeSelected returns true if SelectedPackageTypes is empty or if +// IsPackageTypeSelected returns true if SelectedPackageTypes is empty or if // pkgType is present on SelectedPackageTypes. It returns false otherwise. -func isPackageTypeSelected(pkgType PackageType) bool { +func IsPackageTypeSelected(pkgType PackageType) bool { if len(SelectedPackageTypes) == 0 { return true } @@ -127,9 +145,9 @@ func isPackageTypeSelected(pkgType PackageType) bool { return false } -// isDockerVariantSelected returns true if SelectedDockerVariants is empty or if +// IsDockerVariantSelected returns true if SelectedDockerVariants is empty or if // docVariant is present on SelectedDockerVariants. It returns false otherwise. -func isDockerVariantSelected(docVariant DockerVariant) bool { +func IsDockerVariantSelected(docVariant DockerVariant) bool { if len(SelectedDockerVariants) == 0 { return true } @@ -222,7 +240,7 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "-v") } - args = append(args, MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/package_test.go")) + args = append(args, MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/testing/package_test.go")) if params.HasModules { args = append(args, "--modules") @@ -248,10 +266,6 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "-root-owner") } - if FIPSBuild { - args = append(args, "-fips") - } - args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*")) if out, err := goTest(args...); err != nil { diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 01f27b57297..17682d68c09 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -28,6 +28,7 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon" + "github.com/elastic/elastic-agent/dev-tools/packaging" ) const ( @@ -39,13 +40,13 @@ const ( packageStagingDir = "build/package" // defaultBinaryName specifies the output file for zip and tar.gz. - defaultBinaryName = "{{.Name}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}{{if .FIPS}}-fips{{end}}" + defaultBinaryName = "{{.Name}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" // defaultRootDir is the default name of the root directory contained inside of zip and // tar.gz packages. // NOTE: This uses .BeatName instead of .Name because we wanted the internal // directory to not include "-oss". - defaultRootDir = "{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}{{if .FIPS}}-fips{{end}}" + defaultRootDir = "{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" componentConfigMode os.FileMode = 0600 @@ -105,6 +106,7 @@ type PackageSpec struct { Qualifier string `yaml:"qualifier,omitempty"` // Optional OutputFile string `yaml:"output_file,omitempty"` // Optional ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional + Components []packaging.BinarySpec `yaml:"components"` // Optional: Components required for this package evalContext map[string]interface{} packageDir string @@ -415,6 +417,7 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec { s.packageDir = filepath.Clean(mustExpand(s.packageDir)) } s.evalContext["PackageDir"] = s.packageDir + s.evalContext["fips"] = s.FIPS evaluatedFiles := make(map[string]PackageFile, len(s.Files)) for target, f := range s.Files { diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index b237ebe7900..c37813d40c2 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -336,7 +336,7 @@ func AgentPackageVersion() (string, error) { return BeatQualifiedVersion() } -func PackageManifest() (string, error) { +func PackageManifest(fips bool) (string, error) { packageVersion, err := AgentPackageVersion() if err != nil { @@ -353,14 +353,15 @@ func PackageManifest() (string, error) { return "", fmt.Errorf("retrieving agent commit hash: %w", err) } - return GeneratePackageManifest(BeatName, packageVersion, Snapshot, hash, commitHashShort) + return GeneratePackageManifest(BeatName, packageVersion, Snapshot, hash, commitHashShort, fips) } -func GeneratePackageManifest(beatName, packageVersion string, snapshot bool, fullHash, shortHash string) (string, error) { +func GeneratePackageManifest(beatName, packageVersion string, snapshot bool, fullHash, shortHash string, fips bool) (string, error) { m := v1.NewManifest() m.Package.Version = packageVersion m.Package.Snapshot = snapshot m.Package.Hash = fullHash + m.Package.Fips = fips versionedHomePath := path.Join("data", fmt.Sprintf("%s-%s", beatName, shortHash)) m.Package.VersionedHome = versionedHomePath diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 250f65b4972..d216fffa8ff 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -1,9 +1,218 @@ --- -# This file contains the package specifications for both Community Beats and -# Official Beats. The shared section contains YAML anchors that are used to +# This file contains the package specifications for Elastic Agent + +# List all the available platforms +platforms: &all-platforms + - &linux-amd64 + os: linux + arch: x86_64 + - &linux-arm64 + os: linux + arch: arm64 + - &windows-amd64 + os: windows + arch: x86_64 + - &darwin-amd64 + os: darwin + arch: x86_64 + - &darwin-arm64 + os: darwin + arch: aarch64 + +# List all the package type constants (see dev-tools/mage/pkgcommon/pkgcommon-types.go) +packageTypes: &all-package-types + - &pkg-type-rpm + 1 # RPM + - &pkg-type-deb + 2 # Deb + - &pkg-type-zip + 3 # zip + - &pkg-type-targz + 4 # tar.gz + - &pkg-type-docker + 5 # docker + +# Settings section contains general compiling and packaging settings +settings: + fips: + compile: + cgo: true + env: + GOEXPERIMENT: systemcrypto + tags: + - requirefips + platforms: + - *linux-amd64 + - *linux-arm64 + +# List *all* the components available for packaging in elastic-agent +components: + # general template for new components (all attributes are mandatory) + # - &comp- + # projectName: + # packageName: