Fips packaging

.buildkite/integration.pipeline.yml

@@ -118,6 +118,36 @@ steps:
           imagePrefix: "core-ubuntu-2204-aarch64"
           diskSizeGb: 200
 
+      - label: "Packaging: Containers linux/amd64 FIPS"
+        key: packaging-containers-x86-64-fips
+        env:
+          PACKAGES: "docker"
+          PLATFORMS: "linux/amd64"
+          FIPS: "true"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+          diskSizeGb: 200
+
+      - label: "Packaging: Containers linux/arm64 FIPS"
+        key: packaging-containers-arm64-fips
+        env:
+          PACKAGES: "docker"
+          PLATFORMS: "linux/arm64"
+          FIPS: "true"
+        command: |
+          .buildkite/scripts/steps/integration-package.sh
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "aws"
+          instanceType: "c6g.4xlarge"
+          imagePrefix: "core-ubuntu-2204-aarch64"
+          diskSizeGb: 200
+
   - label: "Serverless integration test"
     key: "serverless-integration-tests"
     depends_on:

.buildkite/pipeline.elastic-agent-binary-dra.yml

@@ -25,7 +25,21 @@ steps:
       env:
         DRA_WORKFLOW: "snapshot"
         PLATFORMS: "linux/amd64 windows/amd64 darwin/amd64"
-
+
+    - label: ":package: linux/amd64 FIPS Elastic-Agent Core Snapshot"
+      commands:
+        - .buildkite/scripts/steps/build-agent-core.sh
+      key: "build-dra-snapshot-x86-fips"
+      artifact_paths:
+        - "build/distributions/**/*"
+      agents:
+        provider: "gcp"
+        machineType: "c2-standard-16"
+      env:
+        DRA_WORKFLOW: "snapshot"
+        PLATFORMS: "linux/amd64"
+        FIPS: "true"
+
     - label: ":package: linux/arm64 darwin/arm64 Elastic-Agent Core Snapshot"
       commands:
         - .buildkite/scripts/steps/build-agent-core.sh
@@ -40,6 +54,20 @@ steps:
         DRA_WORKFLOW: "snapshot"
         PLATFORMS: "linux/arm64 darwin/arm64"
 
+    - label: ":package: linux/arm64 FIPS Elastic-Agent Core Snapshot"
+      commands:
+        - .buildkite/scripts/steps/build-agent-core.sh
+      key: "build-dra-snapshot-arm-fips"
+      artifact_paths:
+        - "build/distributions/**/*"
+      agents:
+        provider: "aws"
+        instanceType: "c6g.4xlarge"
+        imagePrefix: "core-ubuntu-2204-aarch64"
+      env:
+        DRA_WORKFLOW: "snapshot"
+        PLATFORMS: "linux/arm64"
+        FIPS: "true"
     - wait
 
     - label: ":hammer: DRA Publish Elastic-Agent Core Snapshot"
@@ -86,6 +114,21 @@ steps:
         DRA_WORKFLOW: "staging"
         PLATFORMS: "linux/amd64 windows/amd64 darwin/amd64"
 
+    - label: ":package: linux/amd64 FIPS Elastic-Agent Core staging"
+      commands: |
+        source .buildkite/scripts/version_qualifier.sh
+        .buildkite/scripts/steps/build-agent-core.sh
+      key: "build-dra-staging-x86-fips"
+      artifact_paths:
+        - "build/distributions/**/*"
+      agents:
+        provider: "gcp"
+        machineType: "c2-standard-16"
+      env:
+        DRA_WORKFLOW: "staging"
+        PLATFORMS: "linux/amd64"
+        FIPS: "true"
+
     - label: ":package: linux/arm64 darwin/arm64 Elastic-Agent Core staging"
       commands: |
         source .buildkite/scripts/version_qualifier.sh
@@ -101,6 +144,22 @@ steps:
         DRA_WORKFLOW: "dra-core-staging"
         PLATFORMS: "linux/arm64 darwin/arm64"
 
+    - label: ":package: linux/arm64 FIPS Elastic-Agent Core staging"
+      commands: |
+        source .buildkite/scripts/version_qualifier.sh
+        .buildkite/scripts/steps/build-agent-core.sh
+      key: "build-dra-staging-arm-fips"
+      artifact_paths:
+        - "build/distributions/**/*"
+      agents:
+        provider: "aws"
+        instanceType: "c6g.4xlarge"
+        imagePrefix: "core-ubuntu-2204-aarch64"
+      env:
+        DRA_WORKFLOW: "dra-core-staging"
+        PLATFORMS: "linux/arm64"
+        FIPS: "true"
+
     - wait
 
     - label: ":hammer: DRA Publish Elastic-Agent Core staging"

dev-tools/mage/build.go

@@ -18,6 +18,8 @@ import (
 	"github.com/magefile/mage/sh"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
+
+	"github.com/elastic/elastic-agent/dev-tools/packaging"
 )
 
 // BuildArgs are the arguments used for the "build" target and they define how
@@ -73,6 +75,7 @@ func DefaultBuildArgs() BuildArgs {
 	args := BuildArgs{
 		Name: BeatName,
 		CGO:  build.Default.CgoEnabled,
+		Env:  map[string]string{},
 		Vars: map[string]string{
 			elasticAgentModulePath + "/version.buildTime": "{{ date }}",
 			elasticAgentModulePath + "/version.commit":    "{{ commit }}",
@@ -88,8 +91,16 @@ func DefaultBuildArgs() BuildArgs {
 	}
 
 	if FIPSBuild {
-		args.ExtraFlags = append(args.ExtraFlags, "-tags=requirefips")
-		args.CGO = true
+
+		fipsConfig := packaging.Settings().FIPS
+
+		for _, tag := range fipsConfig.Compile.Tags {
+			args.ExtraFlags = append(args.ExtraFlags, "-tags="+tag)
+		}
+		args.CGO = args.CGO || fipsConfig.Compile.CGO
+		for varName, value := range fipsConfig.Compile.Env {
+			args.Env[varName] = value
+		}
 	}
 
 	if DevBuild {
@@ -191,11 +202,6 @@ func Build(params BuildArgs) error {
 		cgoEnabled = "1"
 	}
 
-	if FIPSBuild {
-		cgoEnabled = "1"
-		env["GOEXPERIMENT"] = "systemcrypto"
-	}
-
 	env["CGO_ENABLED"] = cgoEnabled
 
 	// Spec

dev-tools/mage/checksums.go

@@ -9,7 +9,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/magefile/mage/mg"
 	"github.com/otiai10/copy"
@@ -41,68 +40,82 @@ func CopyComponentSpecs(componentName, versionedDropPath string) (string, error)
 	return GetSHA512Hash(targetPath)
 }
 
-// This is a helper function for flattenDependencies that's used when not packaging from a manifest
-func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string, packageVersion string) map[string]string {
-	globExpr := filepath.Join(versionedFlatPath, fmt.Sprintf("*%s*", packageVersion))
-	if mg.Verbose() {
-		log.Printf("Finding files to copy with %s", globExpr)
-	}
-	files, err := filepath.Glob(globExpr)
-	if err != nil {
-		panic(err)
-	}
-	if mg.Verbose() {
-		log.Printf("Validating checksums for %+v", files)
-		log.Printf("--- Copying into %s: %v", versionedDropPath, files)
-	}
-
+// ChecksumsWithoutManifest is a helper function for flattenDependencies that's used when not packaging from a manifest.
+// This function will iterate over the dependencies, resolve *exactly* the package name for each dependency and platform using the passed
+// dependenciesVersion, and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath
+// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform
+// that will be used to compose the package content)
+// ChecksumsWithoutManifest will accumulate the checksums of each component spec that is copied, and return it to the caller.
+func ChecksumsWithoutManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, dependencies []packaging.BinarySpec) map[string]string {
 	checksums := make(map[string]string)
-	for _, f := range files {
+
+	for _, dep := range dependencies {
+
+		if dep.PythonWheel {
+			if mg.Verbose() {
+				log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", dep.ProjectName, dep.BinaryName)
+			}
+			continue
+		}
+
+		if !dep.SupportsPlatform(platform) {
+			log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", dep.ProjectName, dep.BinaryName, platform)
+			continue
+		}
+
+		srcDir := filepath.Join(versionedFlatPath, dep.GetRootDir(dependenciesVersion, platform))
+
+		if mg.Verbose() {
+			log.Printf("Validating checksums for %+v", dep.BinaryName)
+			log.Printf("--- Copying into %s: %v", versionedDropPath, srcDir)
+		}
+
 		options := copy.Options{
 			OnSymlink: func(_ string) copy.SymlinkAction {
 				return copy.Shallow
 			},
 			Sync: true,
 		}
 		if mg.Verbose() {
-			log.Printf("> prepare to copy %s into %s ", f, versionedDropPath)
+			log.Printf("> prepare to copy %s into %s ", srcDir, versionedDropPath)
 		}
 
-		err = copy.Copy(f, versionedDropPath, options)
+		err := copy.Copy(srcDir, versionedDropPath, options)
 		if err != nil {
-			panic(err)
+			panic(fmt.Errorf("copying dependency %s files from %q to %q: %w", dep.BinaryName, srcDir, versionedDropPath, err))
 		}
 
 		// copy spec file for match
-		specName := filepath.Base(f)
-		idx := strings.Index(specName, "-"+packageVersion)
-		if idx != -1 {
-			specName = specName[:idx]
-		}
 		if mg.Verbose() {
-			log.Printf(">>>> Looking to copy spec file: [%s]", specName)
+			log.Printf(">>>> Looking to copy spec file: [%s]", dep.BinaryName)
 		}
 
-		checksum, err := CopyComponentSpecs(specName, versionedDropPath)
+		checksum, err := CopyComponentSpecs(dep.BinaryName, versionedDropPath)
 		if err != nil {
 			panic(err)
 		}
 
-		checksums[specName+ComponentSpecFileSuffix] = checksum
+		checksums[dep.BinaryName+ComponentSpecFileSuffix] = checksum
 	}
 
 	return checksums
 }
 
-// This is a helper function for flattenDependencies that's used when building from a manifest
-func ChecksumsWithManifest(platform, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build) map[string]string {
+// ChecksumsWithManifest is a helper function for flattenDependencies that's used when building from a manifest.
+// This function will iterate over the dependencies, resolve the package name for each dependency and platform using the manifest,
+// (there may be some variability there in case the manifest does not include an exact match for the expected filename),
+// and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath
+// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform
+// that will be used to compose the package content)
+// ChecksumsWithManifest will accumulate the checksums of each component spec that is copied, and return it to the caller.
+func ChecksumsWithManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build, dependencies []packaging.BinarySpec) map[string]string {
 	checksums := make(map[string]string)
 	if manifestResponse == nil {
 		return checksums
 	}
 
 	// Iterate over the external binaries that we care about for packaging agent
-	for _, spec := range packaging.ExpectedBinaries {
+	for _, spec := range dependencies {
 
 		if spec.PythonWheel {
 			if mg.Verbose() {
@@ -124,21 +137,13 @@ func ChecksumsWithManifest(platform, dependenciesVersion string, versionedFlatPa
 			continue
 		}
 
+		rootDir := spec.GetRootDir(manifestPackage.ActualVersion, platform)
+
 		// Combine the package name w/ the versioned flat path
-		fullPath := filepath.Join(versionedFlatPath, manifestPackage.Name)
-
-		// Eliminate the file extensions to get the proper directory
-		// name that we need to copy
-		var dirToCopy string
-		if strings.HasSuffix(fullPath, ".tar.gz") {
-			dirToCopy = fullPath[:strings.LastIndex(fullPath, ".tar.gz")]
-		} else if strings.HasSuffix(fullPath, ".zip") {
-			dirToCopy = fullPath[:strings.LastIndex(fullPath, ".zip")]
-		} else {
-			dirToCopy = fullPath
-		}
+		fullPath := filepath.Join(versionedFlatPath, rootDir)
+
 		if mg.Verbose() {
-			log.Printf(">>>>>>> Calculated directory to copy: [%s]", dirToCopy)
+			log.Printf(">>>>>>> Calculated directory to copy: [%s]", fullPath)
 		}
 
 		// Set copy options
@@ -149,11 +154,11 @@ func ChecksumsWithManifest(platform, dependenciesVersion string, versionedFlatPa
 			Sync: true,
 		}
 		if mg.Verbose() {
-			log.Printf("> prepare to copy %s into %s ", dirToCopy, versionedDropPath)
+			log.Printf("> prepare to copy %s into %s ", fullPath, versionedDropPath)
 		}
 
 		// Do the copy
-		err = copy.Copy(dirToCopy, versionedDropPath, options)
+		err = copy.Copy(fullPath, versionedDropPath, options)
 		if err != nil {
 			panic(err)
 		}

dev-tools/mage/crossbuild.go

@@ -30,11 +30,11 @@ const defaultCrossBuildTarget = "golangCrossBuild"
 var Platforms = BuildPlatforms.Defaults()
 
 // SelectedPackageTypes is the list of package types. If empty, all packages types
-// are considered to be selected (see isPackageTypeSelected).
+// are considered to be selected (see IsPackageTypeSelected).
 var SelectedPackageTypes []PackageType
 
 // SelectedDockerVariants is the list of docker variants. If empty, all docker variants
-// are considered to be selected (see isDockerVariantSelected).
+// are considered to be selected (see IsDockerVariantSelected).
 var SelectedDockerVariants []DockerVariant
 
 func init() {

dev-tools/mage/dockerbuilder.go

@@ -227,9 +227,7 @@ func (b *dockerBuilder) dockerBuild() (string, []string, error) {
 	if b.Snapshot {
 		mainTag = mainTag + "-SNAPSHOT"
 	}
-	if b.FIPS {
-		mainTag = mainTag + "-fips"
-	}
+
 	if repository := b.ExtraVars["repository"]; repository != "" {
 		mainTag = fmt.Sprintf("%s/%s", repository, mainTag)
 	}