Skip to content

Include python wheel packages when moving downloaded dependencies archives #5647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pchila merged 1 commit into from
Oct 1, 2024
Merged

Include python wheel packages when moving downloaded dependencies archives #5647

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 41 additions & 15 deletions magefile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ const (
agentCoreProjectName = "elastic-agent-core"

helmChartPath = "./deploy/helm/elastic-agent"

sha512FileExt = ".sha512"
)

var (
Expand Down Expand Up @@ -947,16 +949,16 @@ func runAgent(ctx context.Context, env map[string]string) error {
func packageAgent(ctx context.Context, platforms []string, dependenciesVersion string, manifestResponse *manifest.Build, agentPackaging, agentBinaryTarget mg.Fn) error {
fmt.Println("--- Package Elastic-Agent")

requiredPackages := []string{}
platformPackageSuffixes := []string{}
for _, p := range platforms {
requiredPackages = append(requiredPackages, manifest.PlatformPackages[p])
platformPackageSuffixes = append(platformPackageSuffixes, manifest.PlatformPackages[p])
}
if mg.Verbose() {
log.Printf("--- Packaging dependenciesVersion[%s], %+v \n", dependenciesVersion, requiredPackages)
log.Printf("--- Packaging dependenciesVersion[%s], %+v \n", dependenciesVersion, platformPackageSuffixes)
}

// download/copy all the necessary dependencies for packaging elastic-agent
archivePath, dropPath := collectPackageDependencies(platforms, dependenciesVersion, requiredPackages)
archivePath, dropPath := collectPackageDependencies(platforms, dependenciesVersion, platformPackageSuffixes)

// cleanup after build
defer os.RemoveAll(archivePath)
Expand All @@ -972,7 +974,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s
defer os.RemoveAll(flatPath)

// extract all dependencies from their archives into flat dir
flattenDependencies(requiredPackages, dependenciesVersion, archivePath, dropPath, flatPath, manifestResponse)
flattenDependencies(platformPackageSuffixes, dependenciesVersion, archivePath, dropPath, flatPath, manifestResponse)

// package agent
log.Println("--- Running packaging function")
Expand All @@ -990,7 +992,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s
// NOTE: after the build is done the caller must:
// - delete archivePath and dropPath contents
// - unset AGENT_DROP_PATH environment variable
func collectPackageDependencies(platforms []string, packageVersion string, requiredPackages []string) (archivePath string, dropPath string) {
func collectPackageDependencies(platforms []string, packageVersion string, platformPackageSuffixes []string) (archivePath string, dropPath string) {

dropPath, found := os.LookupEnv(agentDropPath)

Expand All @@ -1009,7 +1011,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi
if mg.Verbose() {
log.Printf(">> Creating drop-in folder %+v \n", dropPath)
}
archivePath = movePackagesToArchive(dropPath, requiredPackages)
archivePath = movePackagesToArchive(dropPath, platformPackageSuffixes, packageVersion)

if hasSnapshotEnv() {
packageVersion = fmt.Sprintf("%s-SNAPSHOT", packageVersion)
Expand Down Expand Up @@ -1061,7 +1063,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi

packagesCopied := 0

if !requiredPackagesPresent(pwd, b, packageVersion, requiredPackages) {
if !requiredPackagesPresent(pwd, b, packageVersion, platformPackageSuffixes) {
fmt.Printf("--- Package %s\n", pwd)
cmd := exec.Command("mage", "package")
cmd.Dir = pwd
Expand All @@ -1079,7 +1081,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi

// copy to new drop
sourcePath := filepath.Join(pwd, "build", "distributions")
for _, rp := range requiredPackages {
for _, rp := range platformPackageSuffixes {
files, err := filepath.Glob(filepath.Join(sourcePath, "*"+rp+"*"))
if err != nil {
panic(err)
Expand Down Expand Up @@ -1112,7 +1114,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi
}
}
} else {
archivePath = movePackagesToArchive(dropPath, requiredPackages)
archivePath = movePackagesToArchive(dropPath, platformPackageSuffixes, packageVersion)
}
return archivePath, dropPath
}
Expand Down Expand Up @@ -1161,6 +1163,10 @@ func flattenDependencies(requiredPackages []string, packageVersion, archivePath,
}
matches = append(matches, zipMatches...)

if mg.Verbose() {
log.Printf("--- Unfiltered dependencies to flatten in %s : %v", targetPath, matches)
}

// never flatten any python wheels, the packages.yml and docker should handle
// those specifically so that the python wheels are installed into the container
matches = removePythonWheels(matches, packageVersion)
Expand Down Expand Up @@ -1426,7 +1432,7 @@ func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir s
}

// download the SHA to check integrity
artifactSHADownloadPath := filepath.Join(draDownloadDir, pkgName+".sha512")
artifactSHADownloadPath := filepath.Join(draDownloadDir, pkgName+sha512FileExt)
err = manifest.DownloadPackage(errCtx, pkgDesc.ShaURL, artifactSHADownloadPath)
if err != nil {
return fmt.Errorf("downloading SHA for %q: %w", pkgName, err)
Expand Down Expand Up @@ -1564,7 +1570,7 @@ func appendComponentChecksums(versionedDropPath string, checksums map[string]str
}

// movePackagesToArchive Create archive folder and move any pre-existing artifacts into it.
func movePackagesToArchive(dropPath string, requiredPackages []string) string {
func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, packageVersion string) string {
archivePath := filepath.Join(dropPath, "archives")
os.MkdirAll(archivePath, 0755)

Expand All @@ -1580,8 +1586,14 @@ func movePackagesToArchive(dropPath string, requiredPackages []string) string {
matches = append(matches, zipMatches...)

for _, f := range matches {
for _, rp := range requiredPackages {
if !strings.Contains(f, rp) {
for _, packageSuffix := range platformPackageSuffixes {
if mg.Verbose() {
log.Printf("--- Evaluating moving dependency %s to archive path %s\n", f, archivePath)
}
if !strings.Contains(f, packageSuffix) && !isPythonWheelPackage(f, packageVersion) {
if mg.Verbose() {
log.Printf("--- Skipped moving dependency %s to archive path\n", f)
}
continue
}

Expand All @@ -1596,20 +1608,34 @@ func movePackagesToArchive(dropPath string, requiredPackages []string) string {
continue
}

targetPath := filepath.Join(archivePath, rp, filepath.Base(f))
targetPath := filepath.Join(archivePath, packageSuffix, filepath.Base(f))
targetDir := filepath.Dir(targetPath)
if err := os.MkdirAll(targetDir, 0750); err != nil {
fmt.Printf("warning: failed to create directory %s: %s", targetDir, err)
}
if err := os.Rename(f, targetPath); err != nil {
panic(fmt.Errorf("failed renaming file: %w", err))
}
if mg.Verbose() {
log.Printf("--- Moved dependency in archive path %s => %s\n", f, targetPath)
}
}
}

return archivePath
}

func isPythonWheelPackage(f string, packageVersion string) bool {
fileBaseName := filepath.Base(f)
for _, spec := range manifest.ExpectedBinaries {
packageName := spec.GetPackageName(packageVersion, "")
if spec.PythonWheel && (fileBaseName == packageName || fileBaseName == packageName+sha512FileExt) {
return true
}
}
return false
}

func selectedPackageTypes() string {
if len(devtools.SelectedPackageTypes) == 0 {
return ""
Expand Down