diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index f3de81d4eac..e4f09c24735 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -118,6 +118,36 @@ steps: imagePrefix: "core-ubuntu-2204-aarch64" diskSizeGb: 200 + - label: "Packaging: Containers linux/amd64 FIPS" + key: packaging-containers-x86-64-fips + env: + PACKAGES: "docker" + PLATFORMS: "linux/amd64" + FIPS: "true" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + diskSizeGb: 200 + + - label: "Packaging: Containers linux/arm64 FIPS" + key: packaging-containers-arm64-fips + env: + PACKAGES: "docker" + PLATFORMS: "linux/arm64" + FIPS: "true" + command: | + .buildkite/scripts/steps/integration-package.sh + artifact_paths: + - build/distributions/** + agents: + provider: "aws" + instanceType: "c6g.4xlarge" + imagePrefix: "core-ubuntu-2204-aarch64" + diskSizeGb: 200 + - label: "Serverless integration test" key: "serverless-integration-tests" depends_on: diff --git a/.buildkite/pipeline.elastic-agent-binary-dra.yml b/.buildkite/pipeline.elastic-agent-binary-dra.yml index 02bbffe3d07..ea3cfd762e4 100644 --- a/.buildkite/pipeline.elastic-agent-binary-dra.yml +++ b/.buildkite/pipeline.elastic-agent-binary-dra.yml @@ -25,7 +25,21 @@ steps: env: DRA_WORKFLOW: "snapshot" PLATFORMS: "linux/amd64 windows/amd64 darwin/amd64" - + + - label: ":package: linux/amd64 FIPS Elastic-Agent Core Snapshot" + commands: + - .buildkite/scripts/steps/build-agent-core.sh + key: "build-dra-snapshot-x86-fips" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + DRA_WORKFLOW: "snapshot" + PLATFORMS: "linux/amd64" + FIPS: "true" + - label: ":package: linux/arm64 darwin/arm64 Elastic-Agent Core Snapshot" commands: - .buildkite/scripts/steps/build-agent-core.sh @@ -40,6 +54,20 @@ steps: DRA_WORKFLOW: "snapshot" PLATFORMS: "linux/arm64 darwin/arm64" + - label: ":package: linux/arm64 FIPS Elastic-Agent Core Snapshot" + commands: + - .buildkite/scripts/steps/build-agent-core.sh + key: "build-dra-snapshot-arm-fips" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "aws" + instanceType: "c6g.4xlarge" + imagePrefix: "core-ubuntu-2204-aarch64" + env: + DRA_WORKFLOW: "snapshot" + PLATFORMS: "linux/arm64" + FIPS: "true" - wait - label: ":hammer: DRA Publish Elastic-Agent Core Snapshot" @@ -86,6 +114,21 @@ steps: DRA_WORKFLOW: "staging" PLATFORMS: "linux/amd64 windows/amd64 darwin/amd64" + - label: ":package: linux/amd64 FIPS Elastic-Agent Core staging" + commands: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/steps/build-agent-core.sh + key: "build-dra-staging-x86-fips" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + DRA_WORKFLOW: "staging" + PLATFORMS: "linux/amd64" + FIPS: "true" + - label: ":package: linux/arm64 darwin/arm64 Elastic-Agent Core staging" commands: | source .buildkite/scripts/version_qualifier.sh @@ -101,6 +144,22 @@ steps: DRA_WORKFLOW: "dra-core-staging" PLATFORMS: "linux/arm64 darwin/arm64" + - label: ":package: linux/arm64 FIPS Elastic-Agent Core staging" + commands: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/steps/build-agent-core.sh + key: "build-dra-staging-arm-fips" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "aws" + instanceType: "c6g.4xlarge" + imagePrefix: "core-ubuntu-2204-aarch64" + env: + DRA_WORKFLOW: "dra-core-staging" + PLATFORMS: "linux/arm64" + FIPS: "true" + - wait - label: ":hammer: DRA Publish Elastic-Agent Core staging" diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 92da55273b1..c5d879a6510 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -18,6 +18,8 @@ import ( "github.com/magefile/mage/sh" "golang.org/x/text/cases" "golang.org/x/text/language" + + "github.com/elastic/elastic-agent/dev-tools/packaging" ) // BuildArgs are the arguments used for the "build" target and they define how @@ -73,6 +75,7 @@ func DefaultBuildArgs() BuildArgs { args := BuildArgs{ Name: BeatName, CGO: build.Default.CgoEnabled, + Env: map[string]string{}, Vars: map[string]string{ elasticAgentModulePath + "/version.buildTime": "{{ date }}", elasticAgentModulePath + "/version.commit": "{{ commit }}", @@ -88,8 +91,16 @@ func DefaultBuildArgs() BuildArgs { } if FIPSBuild { - args.ExtraFlags = append(args.ExtraFlags, "-tags=requirefips") - args.CGO = true + + fipsConfig := packaging.Settings().FIPS + + for _, tag := range fipsConfig.Compile.Tags { + args.ExtraFlags = append(args.ExtraFlags, "-tags="+tag) + } + args.CGO = args.CGO || fipsConfig.Compile.CGO + for varName, value := range fipsConfig.Compile.Env { + args.Env[varName] = value + } } if DevBuild { @@ -191,11 +202,6 @@ func Build(params BuildArgs) error { cgoEnabled = "1" } - if FIPSBuild { - cgoEnabled = "1" - env["GOEXPERIMENT"] = "systemcrypto" - } - env["CGO_ENABLED"] = cgoEnabled // Spec diff --git a/dev-tools/mage/checksums.go b/dev-tools/mage/checksums.go index d4fbfc55808..7d57c204a87 100644 --- a/dev-tools/mage/checksums.go +++ b/dev-tools/mage/checksums.go @@ -9,7 +9,6 @@ import ( "log" "os" "path/filepath" - "strings" "github.com/magefile/mage/mg" "github.com/otiai10/copy" @@ -41,23 +40,36 @@ func CopyComponentSpecs(componentName, versionedDropPath string) (string, error) return GetSHA512Hash(targetPath) } -// This is a helper function for flattenDependencies that's used when not packaging from a manifest -func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string, packageVersion string) map[string]string { - globExpr := filepath.Join(versionedFlatPath, fmt.Sprintf("*%s*", packageVersion)) - if mg.Verbose() { - log.Printf("Finding files to copy with %s", globExpr) - } - files, err := filepath.Glob(globExpr) - if err != nil { - panic(err) - } - if mg.Verbose() { - log.Printf("Validating checksums for %+v", files) - log.Printf("--- Copying into %s: %v", versionedDropPath, files) - } - +// ChecksumsWithoutManifest is a helper function for flattenDependencies that's used when not packaging from a manifest. +// This function will iterate over the dependencies, resolve *exactly* the package name for each dependency and platform using the passed +// dependenciesVersion, and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath +// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform +// that will be used to compose the package content) +// ChecksumsWithoutManifest will accumulate the checksums of each component spec that is copied, and return it to the caller. +func ChecksumsWithoutManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, dependencies []packaging.BinarySpec) map[string]string { checksums := make(map[string]string) - for _, f := range files { + + for _, dep := range dependencies { + + if dep.PythonWheel { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", dep.ProjectName, dep.BinaryName) + } + continue + } + + if !dep.SupportsPlatform(platform) { + log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", dep.ProjectName, dep.BinaryName, platform) + continue + } + + srcDir := filepath.Join(versionedFlatPath, dep.GetRootDir(dependenciesVersion, platform)) + + if mg.Verbose() { + log.Printf("Validating checksums for %+v", dep.BinaryName) + log.Printf("--- Copying into %s: %v", versionedDropPath, srcDir) + } + options := copy.Options{ OnSymlink: func(_ string) copy.SymlinkAction { return copy.Shallow @@ -65,44 +77,45 @@ func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string Sync: true, } if mg.Verbose() { - log.Printf("> prepare to copy %s into %s ", f, versionedDropPath) + log.Printf("> prepare to copy %s into %s ", srcDir, versionedDropPath) } - err = copy.Copy(f, versionedDropPath, options) + err := copy.Copy(srcDir, versionedDropPath, options) if err != nil { - panic(err) + panic(fmt.Errorf("copying dependency %s files from %q to %q: %w", dep.BinaryName, srcDir, versionedDropPath, err)) } // copy spec file for match - specName := filepath.Base(f) - idx := strings.Index(specName, "-"+packageVersion) - if idx != -1 { - specName = specName[:idx] - } if mg.Verbose() { - log.Printf(">>>> Looking to copy spec file: [%s]", specName) + log.Printf(">>>> Looking to copy spec file: [%s]", dep.BinaryName) } - checksum, err := CopyComponentSpecs(specName, versionedDropPath) + checksum, err := CopyComponentSpecs(dep.BinaryName, versionedDropPath) if err != nil { panic(err) } - checksums[specName+ComponentSpecFileSuffix] = checksum + checksums[dep.BinaryName+ComponentSpecFileSuffix] = checksum } return checksums } -// This is a helper function for flattenDependencies that's used when building from a manifest -func ChecksumsWithManifest(platform, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build) map[string]string { +// ChecksumsWithManifest is a helper function for flattenDependencies that's used when building from a manifest. +// This function will iterate over the dependencies, resolve the package name for each dependency and platform using the manifest, +// (there may be some variability there in case the manifest does not include an exact match for the expected filename), +// and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath +// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform +// that will be used to compose the package content) +// ChecksumsWithManifest will accumulate the checksums of each component spec that is copied, and return it to the caller. +func ChecksumsWithManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build, dependencies []packaging.BinarySpec) map[string]string { checksums := make(map[string]string) if manifestResponse == nil { return checksums } // Iterate over the external binaries that we care about for packaging agent - for _, spec := range packaging.ExpectedBinaries { + for _, spec := range dependencies { if spec.PythonWheel { if mg.Verbose() { @@ -124,21 +137,13 @@ func ChecksumsWithManifest(platform, dependenciesVersion string, versionedFlatPa continue } + rootDir := spec.GetRootDir(manifestPackage.ActualVersion, platform) + // Combine the package name w/ the versioned flat path - fullPath := filepath.Join(versionedFlatPath, manifestPackage.Name) - - // Eliminate the file extensions to get the proper directory - // name that we need to copy - var dirToCopy string - if strings.HasSuffix(fullPath, ".tar.gz") { - dirToCopy = fullPath[:strings.LastIndex(fullPath, ".tar.gz")] - } else if strings.HasSuffix(fullPath, ".zip") { - dirToCopy = fullPath[:strings.LastIndex(fullPath, ".zip")] - } else { - dirToCopy = fullPath - } + fullPath := filepath.Join(versionedFlatPath, rootDir) + if mg.Verbose() { - log.Printf(">>>>>>> Calculated directory to copy: [%s]", dirToCopy) + log.Printf(">>>>>>> Calculated directory to copy: [%s]", fullPath) } // Set copy options @@ -149,11 +154,11 @@ func ChecksumsWithManifest(platform, dependenciesVersion string, versionedFlatPa Sync: true, } if mg.Verbose() { - log.Printf("> prepare to copy %s into %s ", dirToCopy, versionedDropPath) + log.Printf("> prepare to copy %s into %s ", fullPath, versionedDropPath) } // Do the copy - err = copy.Copy(dirToCopy, versionedDropPath, options) + err = copy.Copy(fullPath, versionedDropPath, options) if err != nil { panic(err) } diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index c7340490d41..8c4e70d150e 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -30,11 +30,11 @@ const defaultCrossBuildTarget = "golangCrossBuild" var Platforms = BuildPlatforms.Defaults() // SelectedPackageTypes is the list of package types. If empty, all packages types -// are considered to be selected (see isPackageTypeSelected). +// are considered to be selected (see IsPackageTypeSelected). var SelectedPackageTypes []PackageType // SelectedDockerVariants is the list of docker variants. If empty, all docker variants -// are considered to be selected (see isDockerVariantSelected). +// are considered to be selected (see IsDockerVariantSelected). var SelectedDockerVariants []DockerVariant func init() { diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 8bc7498d9f5..627c665bbcb 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -227,9 +227,7 @@ func (b *dockerBuilder) dockerBuild() (string, []string, error) { if b.Snapshot { mainTag = mainTag + "-SNAPSHOT" } - if b.FIPS { - mainTag = mainTag + "-fips" - } + if repository := b.ExtraVars["repository"]; repository != "" { mainTag = fmt.Sprintf("%s/%s", repository, mainTag) } diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go index 8557e7bd2bc..7c33f08e8b3 100644 --- a/dev-tools/mage/manifest/manifest.go +++ b/dev-tools/mage/manifest/manifest.go @@ -126,7 +126,7 @@ func DownloadManifest(ctx context.Context, manifest string) (Build, error) { // DownloadComponents is going to download a set of components from the given manifest into the destination // dropPath folder in order to later use that folder for packaging -func DownloadComponents(ctx context.Context, manifest string, platforms []string, dropPath string) error { +func DownloadComponents(ctx context.Context, expectedBinaries []packaging.BinarySpec, manifest string, platforms []string, dropPath string) error { manifestResponse, err := DownloadManifest(ctx, manifest) if err != nil { return fmt.Errorf("failed to download remote manifest file %w", err) @@ -145,7 +145,7 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string errGrp, downloadsCtx := errgroup.WithContext(ctx) // for project, pkgs := range expectedProjectPkgs() { - for _, spec := range packaging.ExpectedBinaries { + for _, spec := range expectedBinaries { for _, platform := range platforms { targetPath := filepath.Join(dropPath) err := os.MkdirAll(targetPath, 0755) @@ -187,8 +187,9 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string } type ResolvedPackage struct { - Name string - URLs []string + Name string + ActualVersion string + URLs []string } func ResolveManifestPackage(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) { @@ -206,24 +207,29 @@ func ResolveManifestPackage(project Project, spec packaging.BinarySpec, dependen } return &ResolvedPackage{ - Name: packageName, - URLs: []string{exactMatch.URL, exactMatch.ShaURL, exactMatch.AscURL}, + Name: packageName, + ActualVersion: dependencyVersion, + URLs: []string{exactMatch.URL, exactMatch.ShaURL, exactMatch.AscURL}, }, nil } // If we didn't find it, it may be an Independent Agent Release, where // the opted-in projects will have a patch version one higher than // the rest of the projects, so we "relax" the version constraint + return resolveManifestPackageUsingRelaxedVersion(project, spec, dependencyVersion, platform) +} + +func resolveManifestPackageUsingRelaxedVersion(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) { + // start with the rendered package name + packageName := spec.GetPackageName(dependencyVersion, platform) - // Find the original version in the filename + // Find the original version in the rendered filename versionIndex := strings.Index(packageName, dependencyVersion) if versionIndex == -1 { return nil, fmt.Errorf("no exact match and filename %q does not seem to contain dependencyVersion %q to try a fallback", packageName, dependencyVersion) } - // TODO move relaxVersion to the version package so we can rewrite the version like so - //parseVersion, _ := version.ParseVersion(dependencyVersion) - //parseVersion.GetRelaxedPatchRegexp() + // obtain a regexp from the exact version string that allows for some flexibility on patch version, prerelease and build metadata tokens relaxedVersion, err := relaxVersion(dependencyVersion) if err != nil { return nil, fmt.Errorf("relaxing dependencyVersion %q: %w", dependencyVersion, err) @@ -235,7 +241,7 @@ func ResolveManifestPackage(project Project, spec packaging.BinarySpec, dependen // locate the original version in the filename and substitute the relaxed version regexp, quoting everything around that relaxedPackageName := regexp.QuoteMeta(packageName[:versionIndex]) - relaxedPackageName += relaxedVersion + relaxedPackageName += `(?P` + relaxedVersion + `)` relaxedPackageName += regexp.QuoteMeta(packageName[versionIndex+len(dependencyVersion):]) if mg.Verbose() { @@ -251,18 +257,19 @@ func ResolveManifestPackage(project Project, spec packaging.BinarySpec, dependen if mg.Verbose() { log.Printf(">>>>>>>>>>> Evaluating filename %s", pkgName) } - if relaxedPackageNameRegexp.MatchString(pkgName) { + if submatches := relaxedPackageNameRegexp.FindStringSubmatch(pkgName); len(submatches) > 0 { if mg.Verbose() { log.Printf(">>>>>>>>>>> Found matching packageName for [%s, %s]: %s", project.Branch, project.CommitHash, pkgName) } return &ResolvedPackage{ - Name: pkgName, - URLs: []string{pkg.URL, pkg.ShaURL, pkg.AscURL}, + Name: pkgName, + ActualVersion: submatches[1], + URLs: []string{pkg.URL, pkg.ShaURL, pkg.AscURL}, }, nil } } - return nil, fmt.Errorf("package [%s] not found in project manifest at %s", packageName, project.ExternalArtifactsManifestURL) + return nil, fmt.Errorf("package [%s] not found in project manifest at %s using relaxed version %q", packageName, project.ExternalArtifactsManifestURL, relaxedPackageName) } // versionRegexp is taken from https://semver.org/ (see the FAQ section/Is there a suggested regular expression (RegEx) to check a SemVer string?) @@ -303,12 +310,12 @@ func DownloadPackage(ctx context.Context, downloadUrl string, target string) err } valid := false for _, manifestHost := range AllowedManifestHosts { - if manifestHost == parsedURL.Host { + if manifestHost == parsedURL.Hostname() { valid = true } } if !valid { - log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Host, AllowedManifestHosts) + log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Hostname(), AllowedManifestHosts) return errorNotAllowedManifestURL } cleanUrl := fmt.Sprintf("https://%s%s", parsedURL.Host, parsedURL.Path) diff --git a/dev-tools/mage/manifest/manifest_test.go b/dev-tools/mage/manifest/manifest_test.go index 0d971edf69b..76a0d53cd2e 100644 --- a/dev-tools/mage/manifest/manifest_test.go +++ b/dev-tools/mage/manifest/manifest_test.go @@ -138,7 +138,7 @@ func TestResolveManifestPackage(t *testing.T) { projects := manifestJson.Projects // Verify the component name is in the list of expected packages. - spec, ok := findBinarySpec(tc.binary) + spec, ok := findBinarySpec(t, tc.binary) assert.True(t, ok) if !spec.SupportsPlatform(tc.platform) { @@ -158,8 +158,11 @@ func TestResolveManifestPackage(t *testing.T) { } } -func findBinarySpec(name string) (packaging.BinarySpec, bool) { - for _, spec := range packaging.ExpectedBinaries { +func findBinarySpec(t *testing.T, name string) (packaging.BinarySpec, bool) { + components, err := packaging.Components() + require.NoError(t, err, "error loading components from packages.yml") + + for _, spec := range components { if spec.BinaryName == name { return spec, true } diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index ed93e86344e..2103305e48d 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -34,6 +34,15 @@ func Package() error { // platforms := updateWithDarwinUniversal(Platforms) platforms := Platforms + if mg.Verbose() { + debugSelectedPackageSpecsWithPlatform := make([]string, 0, len(Packages)) + for _, p := range Packages { + debugSelectedPackageSpecsWithPlatform = append(debugSelectedPackageSpecsWithPlatform, fmt.Sprintf("spec %s on %s/%s", p.Spec.Name, p.OS, p.Arch)) + } + + log.Printf("Packaging for platforms %v, packages %v", platforms, debugSelectedPackageSpecsWithPlatform) + } + tasks := make(map[string][]interface{}) for _, target := range platforms { for _, pkg := range Packages { @@ -41,13 +50,19 @@ func Package() error { continue } + // Checks if this package is compatible with the FIPS settings + if pkg.Spec.FIPS != FIPSBuild { + log.Printf("Skipping %s/%s package type because FIPS flag doesn't match [pkg=%v, build=%v]", pkg.Spec.Name, pkg.OS, pkg.Spec.FIPS, FIPSBuild) + continue + } + for _, pkgType := range pkg.Types { - if !isPackageTypeSelected(pkgType) { + if !IsPackageTypeSelected(pkgType) { log.Printf("Skipping %s package type because it is not selected", pkgType) continue } - if pkgType == Docker && !isDockerVariantSelected(pkg.Spec.DockerVariant) { + if pkgType == Docker && !IsDockerVariantSelected(pkg.Spec.DockerVariant) { log.Printf("Skipping %s docker variant type because it is not selected", pkg.Spec.DockerVariant) continue } @@ -80,7 +95,6 @@ func Package() error { spec.OS = target.GOOS() spec.Arch = packageArch spec.Snapshot = Snapshot - spec.FIPS = FIPSBuild spec.evalContext = map[string]interface{}{ "GOOS": target.GOOS(), "GOARCH": target.GOARCH(), @@ -100,6 +114,10 @@ func Package() error { spec = spec.Evaluate() + if mg.Verbose() { + log.Printf("Adding task for packaging %s on %s/%s", spec.Name, target.GOOS(), target.Arch()) + } + tasks[target.GOOS()+"-"+target.Arch()] = append(tasks[target.GOOS()+"-"+target.Arch()], packageBuilder{target, spec, pkgType}.Build) } } @@ -112,9 +130,9 @@ func Package() error { return nil } -// isPackageTypeSelected returns true if SelectedPackageTypes is empty or if +// IsPackageTypeSelected returns true if SelectedPackageTypes is empty or if // pkgType is present on SelectedPackageTypes. It returns false otherwise. -func isPackageTypeSelected(pkgType PackageType) bool { +func IsPackageTypeSelected(pkgType PackageType) bool { if len(SelectedPackageTypes) == 0 { return true } @@ -127,9 +145,9 @@ func isPackageTypeSelected(pkgType PackageType) bool { return false } -// isDockerVariantSelected returns true if SelectedDockerVariants is empty or if +// IsDockerVariantSelected returns true if SelectedDockerVariants is empty or if // docVariant is present on SelectedDockerVariants. It returns false otherwise. -func isDockerVariantSelected(docVariant DockerVariant) bool { +func IsDockerVariantSelected(docVariant DockerVariant) bool { if len(SelectedDockerVariants) == 0 { return true } @@ -248,10 +266,6 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "-root-owner") } - if FIPSBuild { - args = append(args, "-fips") - } - args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*")) if out, err := goTest(args...); err != nil { diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 81aef4444ab..3cca0b06848 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -29,6 +29,7 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon" + "github.com/elastic/elastic-agent/dev-tools/packaging" ) const ( @@ -40,13 +41,13 @@ const ( packageStagingDir = "build/package" // defaultBinaryName specifies the output file for zip and tar.gz. - defaultBinaryName = "{{.Name}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}{{if .FIPS}}-fips{{end}}" + defaultBinaryName = "{{.Name}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" // defaultRootDir is the default name of the root directory contained inside of zip and // tar.gz packages. // NOTE: This uses .BeatName instead of .Name because we wanted the internal // directory to not include "-oss". - defaultRootDir = "{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}{{if .FIPS}}-fips{{end}}" + defaultRootDir = "{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" componentConfigMode os.FileMode = 0600 @@ -107,6 +108,7 @@ type PackageSpec struct { OutputFile string `yaml:"output_file,omitempty"` // Optional ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional ExtraTags []string `yaml:"extra_tags,omitempty"` // Optional + Components []packaging.BinarySpec `yaml:"components"` // Optional: Components required for this package evalContext map[string]interface{} packageDir string @@ -428,6 +430,7 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec { s.packageDir = filepath.Clean(mustExpand(s.packageDir)) } s.evalContext["PackageDir"] = s.packageDir + s.evalContext["fips"] = s.FIPS evaluatedFiles := make(map[string]PackageFile, len(s.Files)) for target, f := range s.Files { diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index ab09281e015..bd3e824275a 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -337,7 +337,7 @@ func AgentPackageVersion() (string, error) { return BeatQualifiedVersion() } -func PackageManifest() (string, error) { +func PackageManifest(fips bool) (string, error) { packageVersion, err := AgentPackageVersion() if err != nil { @@ -359,14 +359,15 @@ func PackageManifest() (string, error) { return "", fmt.Errorf("retrieving agent flavors: %w", err) } - return GeneratePackageManifest(BeatName, packageVersion, Snapshot, hash, commitHashShort, registry) + return GeneratePackageManifest(BeatName, packageVersion, Snapshot, hash, commitHashShort, fips, registry) } -func GeneratePackageManifest(beatName, packageVersion string, snapshot bool, fullHash, shortHash string, flavorsRegistry map[string][]string) (string, error) { +func GeneratePackageManifest(beatName, packageVersion string, snapshot bool, fullHash, shortHash string, fips bool, flavorsRegistry map[string][]string) (string, error) { m := v1.NewManifest() m.Package.Version = packageVersion m.Package.Snapshot = snapshot m.Package.Hash = fullHash + m.Package.Fips = fips versionedHomePath := path.Join("data", fmt.Sprintf("%s-%s", beatName, shortHash)) m.Package.VersionedHome = versionedHomePath diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index e537dad6cd2..9c7ac30449a 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -1,9 +1,207 @@ --- -# This file contains the package specifications for both Community Beats and -# Official Beats. The shared section contains YAML anchors that are used to -# define common parts of the package in order to not repeat ourselves. +# This file contains the package specifications for Elastic Agent +# List all the available platforms +platforms: &all-platforms + - &linux-amd64 + os: linux + arch: x86_64 + - &linux-arm64 + os: linux + arch: arm64 + - &windows-amd64 + os: windows + arch: x86_64 + - &darwin-amd64 + os: darwin + arch: x86_64 + - &darwin-arm64 + os: darwin + arch: aarch64 + +# List all the package type constants (see dev-tools/mage/pkgcommon/pkgcommon-types.go) +packageTypes: &all-package-types + - &pkg-type-rpm + 1 # RPM + - &pkg-type-deb + 2 # Deb + - &pkg-type-zip + 3 # zip + - &pkg-type-targz + 4 # tar.gz + - &pkg-type-docker + 5 # docker + +# Settings section contains general compiling and packaging settings +settings: + fips: + compile: + cgo: true + env: + GOEXPERIMENT: systemcrypto + tags: + - requirefips + platforms: + - *linux-amd64 + - *linux-arm64 + +# List *all* the components available for packaging in elastic-agent +components: + # general template for new components (all attributes are mandatory) + # - &comp- + # projectName: + # packageName: