From e02a29aade55d80ae1948ce4169a3ae8ce5d508a Mon Sep 17 00:00:00 2001 From: Paolo Chila Date: Tue, 1 Oct 2024 12:26:22 +0200 Subject: [PATCH] Include python wheel packages when moving downloaded dependencies archives This commit modifies movePackagesToArchive() function to include python wheel packages introduced with commit 701f8b9 . This solves an issue when packaging docker images using a DROP_PATH env var: prior to this change, the python wheel archives present in DROP_PATH were not moved to `/archive/` failing to create an image for which the Dockerfile expects to install the python wheel package --- magefile.go | 56 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/magefile.go b/magefile.go index 06ef0c1a2c3..03aecaeb620 100644 --- a/magefile.go +++ b/magefile.go @@ -93,6 +93,8 @@ const ( agentCoreProjectName = "elastic-agent-core" helmChartPath = "./deploy/helm/elastic-agent" + + sha512FileExt = ".sha512" ) var ( @@ -947,16 +949,16 @@ func runAgent(ctx context.Context, env map[string]string) error { func packageAgent(ctx context.Context, platforms []string, dependenciesVersion string, manifestResponse *manifest.Build, agentPackaging, agentBinaryTarget mg.Fn) error { fmt.Println("--- Package Elastic-Agent") - requiredPackages := []string{} + platformPackageSuffixes := []string{} for _, p := range platforms { - requiredPackages = append(requiredPackages, manifest.PlatformPackages[p]) + platformPackageSuffixes = append(platformPackageSuffixes, manifest.PlatformPackages[p]) } if mg.Verbose() { - log.Printf("--- Packaging dependenciesVersion[%s], %+v \n", dependenciesVersion, requiredPackages) + log.Printf("--- Packaging dependenciesVersion[%s], %+v \n", dependenciesVersion, platformPackageSuffixes) } // download/copy all the necessary dependencies for packaging elastic-agent - archivePath, dropPath := collectPackageDependencies(platforms, dependenciesVersion, requiredPackages) + archivePath, dropPath := collectPackageDependencies(platforms, dependenciesVersion, platformPackageSuffixes) // cleanup after build defer os.RemoveAll(archivePath) @@ -972,7 +974,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s defer os.RemoveAll(flatPath) // extract all dependencies from their archives into flat dir - flattenDependencies(requiredPackages, dependenciesVersion, archivePath, dropPath, flatPath, manifestResponse) + flattenDependencies(platformPackageSuffixes, dependenciesVersion, archivePath, dropPath, flatPath, manifestResponse) // package agent log.Println("--- Running packaging function") @@ -990,7 +992,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s // NOTE: after the build is done the caller must: // - delete archivePath and dropPath contents // - unset AGENT_DROP_PATH environment variable -func collectPackageDependencies(platforms []string, packageVersion string, requiredPackages []string) (archivePath string, dropPath string) { +func collectPackageDependencies(platforms []string, packageVersion string, platformPackageSuffixes []string) (archivePath string, dropPath string) { dropPath, found := os.LookupEnv(agentDropPath) @@ -1009,7 +1011,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi if mg.Verbose() { log.Printf(">> Creating drop-in folder %+v \n", dropPath) } - archivePath = movePackagesToArchive(dropPath, requiredPackages) + archivePath = movePackagesToArchive(dropPath, platformPackageSuffixes, packageVersion) if hasSnapshotEnv() { packageVersion = fmt.Sprintf("%s-SNAPSHOT", packageVersion) @@ -1061,7 +1063,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi packagesCopied := 0 - if !requiredPackagesPresent(pwd, b, packageVersion, requiredPackages) { + if !requiredPackagesPresent(pwd, b, packageVersion, platformPackageSuffixes) { fmt.Printf("--- Package %s\n", pwd) cmd := exec.Command("mage", "package") cmd.Dir = pwd @@ -1079,7 +1081,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi // copy to new drop sourcePath := filepath.Join(pwd, "build", "distributions") - for _, rp := range requiredPackages { + for _, rp := range platformPackageSuffixes { files, err := filepath.Glob(filepath.Join(sourcePath, "*"+rp+"*")) if err != nil { panic(err) @@ -1112,7 +1114,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi } } } else { - archivePath = movePackagesToArchive(dropPath, requiredPackages) + archivePath = movePackagesToArchive(dropPath, platformPackageSuffixes, packageVersion) } return archivePath, dropPath } @@ -1161,6 +1163,10 @@ func flattenDependencies(requiredPackages []string, packageVersion, archivePath, } matches = append(matches, zipMatches...) + if mg.Verbose() { + log.Printf("--- Unfiltered dependencies to flatten in %s : %v", targetPath, matches) + } + // never flatten any python wheels, the packages.yml and docker should handle // those specifically so that the python wheels are installed into the container matches = removePythonWheels(matches, packageVersion) @@ -1426,7 +1432,7 @@ func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir s } // download the SHA to check integrity - artifactSHADownloadPath := filepath.Join(draDownloadDir, pkgName+".sha512") + artifactSHADownloadPath := filepath.Join(draDownloadDir, pkgName+sha512FileExt) err = manifest.DownloadPackage(errCtx, pkgDesc.ShaURL, artifactSHADownloadPath) if err != nil { return fmt.Errorf("downloading SHA for %q: %w", pkgName, err) @@ -1564,7 +1570,7 @@ func appendComponentChecksums(versionedDropPath string, checksums map[string]str } // movePackagesToArchive Create archive folder and move any pre-existing artifacts into it. -func movePackagesToArchive(dropPath string, requiredPackages []string) string { +func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, packageVersion string) string { archivePath := filepath.Join(dropPath, "archives") os.MkdirAll(archivePath, 0755) @@ -1580,8 +1586,14 @@ func movePackagesToArchive(dropPath string, requiredPackages []string) string { matches = append(matches, zipMatches...) for _, f := range matches { - for _, rp := range requiredPackages { - if !strings.Contains(f, rp) { + for _, packageSuffix := range platformPackageSuffixes { + if mg.Verbose() { + log.Printf("--- Evaluating moving dependency %s to archive path %s\n", f, archivePath) + } + if !strings.Contains(f, packageSuffix) && !isPythonWheelPackage(f, packageVersion) { + if mg.Verbose() { + log.Printf("--- Skipped moving dependency %s to archive path\n", f) + } continue } @@ -1596,7 +1608,7 @@ func movePackagesToArchive(dropPath string, requiredPackages []string) string { continue } - targetPath := filepath.Join(archivePath, rp, filepath.Base(f)) + targetPath := filepath.Join(archivePath, packageSuffix, filepath.Base(f)) targetDir := filepath.Dir(targetPath) if err := os.MkdirAll(targetDir, 0750); err != nil { fmt.Printf("warning: failed to create directory %s: %s", targetDir, err) @@ -1604,12 +1616,26 @@ func movePackagesToArchive(dropPath string, requiredPackages []string) string { if err := os.Rename(f, targetPath); err != nil { panic(fmt.Errorf("failed renaming file: %w", err)) } + if mg.Verbose() { + log.Printf("--- Moved dependency in archive path %s => %s\n", f, targetPath) + } } } return archivePath } +func isPythonWheelPackage(f string, packageVersion string) bool { + fileBaseName := filepath.Base(f) + for _, spec := range manifest.ExpectedBinaries { + packageName := spec.GetPackageName(packageVersion, "") + if spec.PythonWheel && (fileBaseName == packageName || fileBaseName == packageName+sha512FileExt) { + return true + } + } + return false +} + func selectedPackageTypes() string { if len(devtools.SelectedPackageTypes) == 0 { return ""