From 77dd2a5e359d49dba442342d41ec01914384c2e9 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 21 Aug 2024 11:37:40 -0400 Subject: [PATCH 1/9] Add ability to select docker variant. --- dev-tools/mage/crossbuild.go | 16 ++++ dev-tools/mage/dockerbuilder.go | 8 +- dev-tools/mage/dockervariants.go | 78 +++++++++++++++++++ dev-tools/mage/pkg.go | 20 +++++ dev-tools/mage/pkgspecs.go | 11 +++ dev-tools/mage/pkgtypes.go | 23 ++---- dev-tools/packaging/packages.yml | 20 ++--- .../docker/Dockerfile.elastic-agent.tmpl | 8 +- 8 files changed, 149 insertions(+), 35 deletions(-) create mode 100644 dev-tools/mage/dockervariants.go diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 6efd816b239..b245c597138 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -33,6 +33,10 @@ var Platforms = BuildPlatforms.Defaults() // are considered to be selected (see isPackageTypeSelected). var SelectedPackageTypes []PackageType +// SelectedDockerVariants is the list of docker variants. If empty, all docker variants +// are considered to be selected (see isDockerVariantSelected). +var SelectedDockerVariants []DockerVariant + func init() { // Allow overriding via PLATFORMS. if expression := os.Getenv("PLATFORMS"); len(expression) > 0 { @@ -50,6 +54,18 @@ func init() { SelectedPackageTypes = append(SelectedPackageTypes, p) } } + + // Allow overriding via DOCKER_VARIANTS. + if dockerVariants := os.Getenv("DOCKER_VARIANTS"); len(dockerVariants) > 0 { + for _, docvariant := range strings.Split(dockerVariants, ",") { + var v DockerVariant + err := v.UnmarshalText([]byte(docvariant)) + if err != nil { + continue + } + SelectedDockerVariants = append(SelectedDockerVariants, v) + } + } } // CrossBuildOption defines an option to the CrossBuild target. diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 87d9e632df1..9948a5dc97b 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -28,17 +28,12 @@ type dockerBuilder struct { } func newDockerBuilder(spec PackageSpec) (*dockerBuilder, error) { - imageName, err := spec.ImageName() - if err != nil { - return nil, err - } - buildDir := filepath.Join(spec.packageDir, "docker-build") beatDir := filepath.Join(buildDir, "beat") return &dockerBuilder{ PackageSpec: spec, - imageName: imageName, + imageName: spec.ImageName(), buildDir: buildDir, beatDir: beatDir, }, nil @@ -117,6 +112,7 @@ func (b *dockerBuilder) prepareBuild() error { data := map[string]interface{}{ "ExposePorts": b.exposePorts(), "ModulesDirs": b.modulesDirs(), + "Variant": b.DockerVariant.String(), } err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { diff --git a/dev-tools/mage/dockervariants.go b/dev-tools/mage/dockervariants.go new file mode 100644 index 00000000000..c03f3ba3992 --- /dev/null +++ b/dev-tools/mage/dockervariants.go @@ -0,0 +1,78 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package mage + +import ( + "fmt" + "strings" +) + +const ( + undefined = "undefined" + basic = "basic" + ubi = "ubi" + wolfi = "wolfi" + complete = "complete" + cloud = "cloud" +) + +// DockerVariant defines the docker variant to build. +type DockerVariant int + +// List of possible docker variants. +const ( + Undefined = iota + Basic + UBI + Wolfi + Complete + Cloud +) + +// String returns the name of the docker variant type. +func (typ DockerVariant) String() string { + switch typ { + case Undefined: + return undefined + case Basic: + return basic + case UBI: + return ubi + case Wolfi: + return wolfi + case Complete: + return complete + case Cloud: + return cloud + default: + return invalid + } +} + +// MarshalText returns the text representation of DockerVariant. +func (typ DockerVariant) MarshalText() ([]byte, error) { + return []byte(typ.String()), nil +} + +// UnmarshalText returns a DockerVariant based on the given text. +func (typ *DockerVariant) UnmarshalText(text []byte) error { + switch strings.ToLower(string(text)) { + case "": + *typ = Undefined + case basic: + *typ = Basic + case ubi: + *typ = UBI + case wolfi: + *typ = Wolfi + case complete: + *typ = Complete + case cloud: + *typ = Cloud + default: + return fmt.Errorf("unknown docker variant: %v", string(text)) + } + return nil +} diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index b13e3d7390f..f728d2b7bd7 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -47,6 +47,11 @@ func Package() error { continue } + if pkgType == Docker && !isDockerVariantSelected(pkg.Spec.DockerVariant) { + log.Printf("Skipping %s docker variant type because it is not selected", pkg.Spec.DockerVariant) + continue + } + if target.Name == "linux/arm64" && pkgType == Docker && runtime.GOARCH != "arm64" { log.Printf("Skipping Docker package type because build host isn't arm") continue @@ -121,6 +126,21 @@ func isPackageTypeSelected(pkgType PackageType) bool { return false } +// isDockerVariantSelected returns true if SelectedDockerVariants is empty or if +// docVariant is present on SelectedDockerVariants. It returns false otherwise. +func isDockerVariantSelected(docVariant DockerVariant) bool { + if len(SelectedDockerVariants) == 0 { + return true + } + + for _, v := range SelectedDockerVariants { + if v == docVariant { + return true + } + } + return false +} + type packageBuilder struct { Platform BuildPlatform Spec PackageSpec diff --git a/dev-tools/mage/pkgspecs.go b/dev-tools/mage/pkgspecs.go index c3450c8525f..165984d8735 100644 --- a/dev-tools/mage/pkgspecs.go +++ b/dev-tools/mage/pkgspecs.go @@ -154,5 +154,16 @@ func LoadSpecs(files ...string) (map[string][]OSPackageArgs, error) { return nil, fmt.Errorf("failed to unmarshal spec data: %w", err) } + // verify that the package specification sets the docker variant + for specName, specs := range packages.Specs { + for _, spec := range specs { + for _, pkgType := range spec.Types { + if pkgType == Docker && spec.Spec.DockerVariant == Undefined { + return nil, fmt.Errorf("%s defined a package spec for docker without a docker_variant set", specName) + } + } + } + } + return packages.Specs, nil } diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index fe2d63750ce..ad3892cb198 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -91,6 +91,7 @@ type PackageSpec struct { License string `yaml:"license,omitempty"` URL string `yaml:"url,omitempty"` Description string `yaml:"description,omitempty"` + DockerVariant DockerVariant `yaml:"docker_variant,omitempty"` PreInstallScript string `yaml:"pre_install_script,omitempty"` PostInstallScript string `yaml:"post_install_script,omitempty"` PostRmScript string `yaml:"post_rm_script,omitempty"` @@ -271,11 +272,7 @@ func (typ PackageType) AddFileExtension(file string) string { func (typ PackageType) PackagingDir(home string, target BuildPlatform, spec PackageSpec) (string, error) { root := home if typ == Docker { - imageName, err := spec.ImageName() - if err != nil { - return "", err - } - root = filepath.Join(root, imageName) + root = filepath.Join(root, spec.ImageName()) } targetPath := typ.AddFileExtension(spec.Name + "-" + target.GOOS() + "-" + target.Arch()) @@ -467,17 +464,13 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec { return s } -// ImageName computes the image name from the spec. A template for the image -// name can be configured by adding image_name to extra_vars. -func (s PackageSpec) ImageName() (string, error) { - if name := s.ExtraVars["image_name"]; name != "" { - imageName, err := s.Expand(name) - if err != nil { - return "", fmt.Errorf("failed to expand image_name: %w", err) - } - return imageName, nil +// ImageName computes the image name from the spec. +func (s PackageSpec) ImageName() string { + if s.DockerVariant == Basic { + // no suffix for basic docker variant + return s.Name } - return s.Name, nil + return fmt.Sprintf("%s-%s", s.Name, s.DockerVariant) } func copyInstallScript(spec PackageSpec, script string, local *string) error { diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 100a1186b07..786636b0a73 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -223,31 +223,31 @@ shared: buildFrom: '--platform=linux/arm64 cgr.dev/chainguard/wolfi-base' - &docker_ubuntu_spec + docker_variant: 'basic' extra_vars: from: '--platform=linux/amd64 ubuntu:20.04' - image_name: '{{.BeatName}}' - &docker_ubuntu_arm_spec + docker_variant: 'basic' extra_vars: from: '--platform=linux/arm64 ubuntu:20.04' - image_name: '{{.BeatName}}' - &docker_ubi_spec + docker_variant: 'ubi' extra_vars: from: '--platform=linux/amd64 docker.elastic.co/ubi9/ubi-minimal' - image_name: '{{.BeatName}}-ubi' - &docker_ubi_arm_spec + docker_variant: 'ubi' extra_vars: from: '--platform=linux/arm64 docker.elastic.co/ubi9/ubi-minimal' - image_name: '{{.BeatName}}-ubi' - &docker_wolfi_spec + docker_variant: 'wolfi' extra_vars: from: '--platform=linux/amd64 cgr.dev/chainguard/wolfi-base' - image_name: '{{.BeatName}}-wolfi' - &docker_wolfi_arm_spec + docker_variant: 'wolfi' extra_vars: from: '--platform=linux/arm64 cgr.dev/chainguard/wolfi-base' - image_name: '{{.BeatName}}-wolfi' - &docker_elastic_spec extra_vars: @@ -275,9 +275,10 @@ shared: {{ commit }} mode: 0644 + # cloud build to beats-ci repository - &agent_docker_cloud_spec + docker_variant: 'cloud' extra_vars: - image_name: '{{.BeatName}}-cloud' repository: 'docker.elastic.co/beats-ci' files: 'data/cloud_downloads/filebeat.sh': @@ -290,11 +291,10 @@ shared: source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0755 - # not different to the default image, kept for backwards-compatibility + # includes nodejs with @elastic/synthetics - &agent_docker_complete_spec <<: *agent_docker_spec - extra_vars: - image_name: '{{.BeatName}}-complete' + docker_variant: 'complete' # Deb/RPM spec for community beats. - &deb_rpm_spec diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 9ccb772633d..0e04c3f9ebd 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -43,7 +43,7 @@ RUN true && \ chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \ {{- end }} -{{- if contains .image_name "-cloud" }} +{{- if eq .Variant "cloud" }} mkdir -p /opt/agentbeat /opt/filebeat /opt/metricbeat && \ cp -f {{ $beatHome }}/data/cloud_downloads/filebeat.sh /opt/filebeat/filebeat && \ chmod +x /opt/filebeat/filebeat && \ @@ -170,14 +170,14 @@ RUN mkdir /licenses COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses -{{- if contains .image_name "-cloud" }} +{{- if eq .Variant "cloud" }} COPY --from=home /opt /opt # Generate folder for a stub command that will be overwritten at runtime RUN mkdir /app && \ chown {{ .user }}:{{ .user }} /app {{- end }} -{{- if (and (contains .image_name "-complete") (contains .from "ubuntu")) }} +{{- if (and (eq .Variant "complete") (contains .from "ubuntu")) }} USER root ENV NODE_PATH={{ $beatHome }}/.node RUN echo \ @@ -254,7 +254,7 @@ ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/ WORKDIR {{ $beatHome }} -{{- if contains .image_name "-cloud" }} +{{- if eq .Variant "cloud" }} ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/app/apm.sh"] # Generate a stub command that will be overwritten at runtime From d30bfb2f46f8dce9beb40d0dc85222915db6f586 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Fri, 23 Aug 2024 12:56:19 -0400 Subject: [PATCH 2/9] Add elastic-agent-service container to packaging. --- dev-tools/mage/checksums.go | 10 +-- dev-tools/mage/dockervariants.go | 6 ++ dev-tools/mage/manifest/manifest.go | 62 +++++++++++-------- dev-tools/mage/manifest/manifest_test.go | 17 +++-- dev-tools/packaging/files/linux/connectors.py | 10 +++ dev-tools/packaging/packages.yml | 43 +++++++++++++ .../docker/Dockerfile.elastic-agent.tmpl | 13 +++- magefile.go | 56 +++++++++++------ specs/connectors.spec.yml | 17 +++++ 9 files changed, 176 insertions(+), 58 deletions(-) create mode 100755 dev-tools/packaging/files/linux/connectors.py create mode 100644 specs/connectors.spec.yml diff --git a/dev-tools/mage/checksums.go b/dev-tools/mage/checksums.go index fbe262f48a6..5e4cb444981 100644 --- a/dev-tools/mage/checksums.go +++ b/dev-tools/mage/checksums.go @@ -108,9 +108,9 @@ func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, ver // Only care about packages that match the required package constraint (os/arch) if strings.Contains(pkgName, requiredPackage) { // Iterate over the external binaries that we care about for packaging agent - for binary := range manifest.ExpectedBinaries { + for _, spec := range manifest.ExpectedBinaries { // If the individual package doesn't match the expected prefix, then continue - if !strings.HasPrefix(pkgName, binary) { + if !strings.HasPrefix(pkgName, spec.BinaryName) { continue } @@ -215,14 +215,14 @@ func getComponentVersion(componentName string, requiredPackage string, component // Iterate over all the packages in the component project for pkgName := range componentProject.Packages { // Only care about the external binaries that we want to package - for binary, project := range manifest.ExpectedBinaries { + for _, spec := range manifest.ExpectedBinaries { // If the given component name doesn't match the external binary component, skip - if componentName != project.Name { + if componentName != spec.ProjectName { continue } // Split the package name on the binary name prefix plus a dash - firstSplit := strings.Split(pkgName, binary+"-") + firstSplit := strings.Split(pkgName, spec.BinaryName+"-") if len(firstSplit) < 2 { continue } diff --git a/dev-tools/mage/dockervariants.go b/dev-tools/mage/dockervariants.go index c03f3ba3992..c72dfd778c2 100644 --- a/dev-tools/mage/dockervariants.go +++ b/dev-tools/mage/dockervariants.go @@ -16,6 +16,7 @@ const ( wolfi = "wolfi" complete = "complete" cloud = "cloud" + service = "service" ) // DockerVariant defines the docker variant to build. @@ -29,6 +30,7 @@ const ( Wolfi Complete Cloud + Service ) // String returns the name of the docker variant type. @@ -46,6 +48,8 @@ func (typ DockerVariant) String() string { return complete case Cloud: return cloud + case Service: + return service default: return invalid } @@ -71,6 +75,8 @@ func (typ *DockerVariant) UnmarshalText(text []byte) error { *typ = Complete case cloud: *typ = Cloud + case service: + *typ = Service default: return fmt.Errorf("unknown docker variant: %v", string(text)) } diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go index 94a521ae369..f74a1898aed 100644 --- a/dev-tools/mage/manifest/manifest.go +++ b/dev-tools/mage/manifest/manifest.go @@ -94,21 +94,24 @@ var PlatformPackages = map[string]string{ // ExpectedBinaries is a map of binaries agent needs to their project in the unified-release manager. // The project names are those used in the "projects" list in the unified release manifest. // See the sample manifests in the testdata directory. -var ExpectedBinaries = map[string]BinarySpec{ - "agentbeat": {Name: "beats", Platforms: AllPlatforms}, - "apm-server": {Name: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}}, - "cloudbeat": {Name: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, - "cloud-defend": {Name: "cloud-defend", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, - "endpoint-security": {Name: "endpoint-dev", Platforms: AllPlatforms}, - "fleet-server": {Name: "fleet-server", Platforms: AllPlatforms}, - "pf-elastic-collector": {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, - "pf-elastic-symbolizer": {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, - "pf-host-agent": {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, +var ExpectedBinaries = []BinarySpec{ + {BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms}, + {BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}}, + {BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, + {BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true}, + {BinaryName: "cloud-defend", ProjectName: "cloud-defend", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, + {BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms}, + {BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms}, + {BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, + {BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, + {BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}}, } type BinarySpec struct { - Name string - Platforms []Platform + BinaryName string + ProjectName string + Platforms []Platform + PythonWheel bool } func (proj BinarySpec) SupportsPlatform(platform string) bool { @@ -120,6 +123,13 @@ func (proj BinarySpec) SupportsPlatform(platform string) bool { return false } +func (proj BinarySpec) GetPackageName(version string, platform string) string { + if proj.PythonWheel { + return fmt.Sprintf("%s-%s.zip", proj.BinaryName, version) + } + return fmt.Sprintf("%s-%s-%s", proj.BinaryName, version, PlatformPackages[platform]) +} + type Platform struct { OS string Arch string @@ -188,27 +198,27 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string errGrp, downloadsCtx := errgroup.WithContext(ctx) // for project, pkgs := range expectedProjectPkgs() { - for binary, project := range ExpectedBinaries { + for _, spec := range ExpectedBinaries { for _, platform := range platforms { targetPath := filepath.Join(dropPath) err := os.MkdirAll(targetPath, 0755) if err != nil { return fmt.Errorf("failed to create directory %s", targetPath) } - log.Printf("+++ Prepare to download project [%s] for [%s]", project.Name, platform) + log.Printf("+++ Prepare to download [%s] project [%s] for [%s]", spec.BinaryName, spec.ProjectName, platform) - if !project.SupportsPlatform(platform) { - log.Printf(">>>>>>>>> Binary [%s] does not support platform [%s] ", binary, platform) + if !spec.SupportsPlatform(platform) { + log.Printf(">>>>>>>>> Binary [%s] does not support platform [%s] ", spec.BinaryName, platform) continue } - pkgURL, err := resolveManifestPackage(projects[project.Name], binary, PlatformPackages[platform], majorMinorPatchVersion) + pkgURL, err := resolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform) if err != nil { return err } for _, p := range pkgURL { - log.Printf(">>>>>>>>> Downloading [%s] [%s] ", binary, p) + log.Printf(">>>>>>>>> Downloading [%s] [%s] ", spec.BinaryName, p) pkgFilename := path.Base(p) downloadTarget := filepath.Join(targetPath, pkgFilename) if _, err := os.Stat(downloadTarget); err != nil { @@ -229,35 +239,35 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string return nil } -func resolveManifestPackage(project Project, binary string, platformPkg string, version string) ([]string, error) { +func resolveManifestPackage(project Project, spec BinarySpec, version string, platform string) ([]string, error) { var val Package var ok bool // Try the normal/easy case first - packageName := fmt.Sprintf("%s-%s-%s", binary, version, platformPkg) + packageName := spec.GetPackageName(version, platform) val, ok = project.Packages[packageName] if !ok { // If we didn't find it, it may be an Independent Agent Release, where // the opted-in projects will have a patch version one higher than // the rest of the projects, so we need to seek that out if mg.Verbose() { - log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", binary, platformPkg) + log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", spec.BinaryName, PlatformPackages[platform]) } var foundIt bool for pkgName := range project.Packages { - if strings.HasPrefix(pkgName, binary) { - firstSplit := strings.Split(pkgName, binary+"-") + if strings.HasPrefix(pkgName, spec.BinaryName) { + firstSplit := strings.Split(pkgName, spec.BinaryName+"-") if len(firstSplit) < 2 { continue } secondHalf := firstSplit[1] // Make sure we're finding one w/ the same required package type - if strings.Contains(secondHalf, platformPkg) { + if strings.Contains(secondHalf, PlatformPackages[platform]) { // Split again after the version with the required package string - secondSplit := strings.Split(secondHalf, "-"+platformPkg) + secondSplit := strings.Split(secondHalf, "-"+PlatformPackages[platform]) if len(secondSplit) < 2 { continue } @@ -269,7 +279,7 @@ func resolveManifestPackage(project Project, binary string, platformPkg string, } // Create a project/package key with the package, derived version, and required package - foundPkgKey := fmt.Sprintf("%s-%s-%s", binary, pkgVersion, platformPkg) + foundPkgKey := fmt.Sprintf("%s-%s-%s", spec.BinaryName, pkgVersion, PlatformPackages[platform]) if mg.Verbose() { log.Printf(">>>>>>>>>>> Looking for project package key: [%s]", foundPkgKey) } diff --git a/dev-tools/mage/manifest/manifest_test.go b/dev-tools/mage/manifest/manifest_test.go index 6951f2dc708..dfe7d6f1e42 100644 --- a/dev-tools/mage/manifest/manifest_test.go +++ b/dev-tools/mage/manifest/manifest_test.go @@ -135,15 +135,15 @@ func TestResolveManifestPackage(t *testing.T) { projects := manifestJson.Projects // Verify the component name is in the list of expected packages. - project, ok := ExpectedBinaries[tc.binary] + spec, ok := findBinarySpec(tc.binary) assert.True(t, ok) - if !project.SupportsPlatform(tc.platform) { - t.Logf("Project %s does not support platform %s", project.Name, tc.platform) + if !spec.SupportsPlatform(tc.platform) { + t.Logf("Project %s does not support platform %s", spec.ProjectName, tc.platform) return } - urlList, err := resolveManifestPackage(projects[tc.projectName], tc.binary, PlatformPackages[tc.platform], manifestJson.Version) + urlList, err := resolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform) require.NoError(t, err) assert.Len(t, urlList, 3) @@ -153,3 +153,12 @@ func TestResolveManifestPackage(t *testing.T) { }) } } + +func findBinarySpec(name string) (BinarySpec, bool) { + for _, spec := range ExpectedBinaries { + if spec.BinaryName == name { + return spec, true + } + } + return BinarySpec{}, false +} diff --git a/dev-tools/packaging/files/linux/connectors.py b/dev-tools/packaging/files/linux/connectors.py new file mode 100755 index 00000000000..7f9dcbdc552 --- /dev/null +++ b/dev-tools/packaging/files/linux/connectors.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +import time + +def main(): + while True: + time.sleep(1) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 786636b0a73..24fcaee3fcf 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -291,6 +291,20 @@ shared: source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0755 + # service build is based on previous cloud variant + - &agent_docker_service_spec + docker_variant: 'service' + files: + 'data/service_downloads/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip': + source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip' + mode: 0755 + 'data/{{.BeatName}}-{{ commit_short }}/components/connectors': + source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/connectors.py' + mode: 0755 + 'data/{{.BeatName}}-{{ commit_short }}/components/connectors.spec.yml': + source: '{{ elastic_beats_dir }}/specs/connectors.spec.yml' + mode: 0644 + # includes nodejs with @elastic/synthetics - &agent_docker_complete_spec <<: *agent_docker_spec @@ -997,6 +1011,35 @@ specs: files: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + #### Service specific docker images #### + - os: linux + arch: amd64 + types: [ docker ] + spec: + <<: *agent_docker_spec + # The service image is always based on Wolfi + <<: *docker_wolfi_spec + <<: *docker_builder_spec + <<: *agent_docker_cloud_spec + <<: *agent_docker_service_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + - os: linux + arch: arm64 + types: [ docker ] + spec: + <<: *agent_docker_spec + # The service image is always based on Wolfi + <<: *docker_wolfi_arm_spec + <<: *docker_builder_arm_spec + <<: *agent_docker_cloud_spec + <<: *agent_docker_service_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} - os: linux arch: amd64 types: [docker] diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 0e04c3f9ebd..967a331d003 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -43,7 +43,7 @@ RUN true && \ chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \ {{- end }} -{{- if eq .Variant "cloud" }} +{{- if or (eq .Variant "cloud") (eq .Variant "service") }} mkdir -p /opt/agentbeat /opt/filebeat /opt/metricbeat && \ cp -f {{ $beatHome }}/data/cloud_downloads/filebeat.sh /opt/filebeat/filebeat && \ chmod +x /opt/filebeat/filebeat && \ @@ -170,13 +170,20 @@ RUN mkdir /licenses COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses -{{- if eq .Variant "cloud" }} +{{- if or (eq .Variant "cloud") (eq .Variant "service") }} COPY --from=home /opt /opt # Generate folder for a stub command that will be overwritten at runtime RUN mkdir /app && \ chown {{ .user }}:{{ .user }} /app {{- end }} +{{- if eq .Variant "service" }} +RUN apk add --no-cache python-3.12 py3.12-pip && \ + pip install {{ $beatHome }}/data/service_downloads/connectors-*.zip && \ + rm -rf {{ $beatHome }}/data/service_downloads && \ + chmod 0755 {{ $beatHome }}/data/elastic-agent-*/components/connectors +{{- end }} + {{- if (and (eq .Variant "complete") (contains .from "ubuntu")) }} USER root ENV NODE_PATH={{ $beatHome }}/.node @@ -254,7 +261,7 @@ ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/ WORKDIR {{ $beatHome }} -{{- if eq .Variant "cloud" }} +{{- if or (eq .Variant "cloud") (eq .Variant "service") }} ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/app/apm.sh"] # Generate a stub command that will be overwritten at runtime diff --git a/magefile.go b/magefile.go index 55be128a0e6..d9dbbe452d4 100644 --- a/magefile.go +++ b/magefile.go @@ -488,11 +488,6 @@ func DownloadManifest(ctx context.Context) error { return errAtLeastOnePlatform } - var requiredPackages []string - for _, p := range platforms { - requiredPackages = append(requiredPackages, manifest.PlatformPackages[p]) - } - if e := manifest.DownloadComponents(ctx, devtools.ManifestURL, platforms, dropPath); e != nil { return fmt.Errorf("failed to download the manifest file, %w", e) } @@ -550,16 +545,9 @@ func FixDRADockerArtifacts() error { return nil } -func getPackageName(beat, version, pkg string) (string, string) { - if hasSnapshotEnv() { - version += "-SNAPSHOT" - } - return version, fmt.Sprintf("%s-%s-%s", beat, version, pkg) -} - func requiredPackagesPresent(basePath, beat, version string, requiredPackages []string) bool { for _, pkg := range requiredPackages { - _, packageName := getPackageName(beat, version, pkg) + packageName := fmt.Sprintf("%s-%s-%s", beat, version, pkg) path := filepath.Join(basePath, "build", "distributions", packageName) if _, err := os.Stat(path); err != nil { @@ -974,6 +962,10 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi } archivePath = movePackagesToArchive(dropPath, requiredPackages) + if hasSnapshotEnv() { + packageVersion = fmt.Sprintf("%s-SNAPSHOT", packageVersion) + } + os.Setenv(agentDropPath, dropPath) if devtools.ExternalBuild == true { @@ -989,17 +981,16 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi errGroup, ctx := errgroup.WithContext(context.Background()) completedDownloads := &atomic.Int32{} - for binary, project := range manifest.ExpectedBinaries { + for _, spec := range manifest.ExpectedBinaries { for _, platform := range platforms { - if !project.SupportsPlatform(platform) { - fmt.Printf("--- Binary %s does not support %s, download skipped\n", binary, platform) + if !spec.SupportsPlatform(platform) { + fmt.Printf("--- Binary %s does not support %s, download skipped\n", spec.BinaryName, platform) continue } - reqPackage := manifest.PlatformPackages[platform] - targetPath := filepath.Join(archivePath, reqPackage) + targetPath := filepath.Join(archivePath, manifest.PlatformPackages[platform]) os.MkdirAll(targetPath, 0755) - newVersion, packageName := getPackageName(binary, packageVersion, reqPackage) - errGroup.Go(downloadBinary(ctx, project.Name, packageName, binary, platform, newVersion, targetPath, completedDownloads)) + packageName := spec.GetPackageName(packageVersion, platform) + errGroup.Go(downloadBinary(ctx, spec.ProjectName, packageName, spec.BinaryName, platform, packageVersion, targetPath, completedDownloads)) } } @@ -1077,6 +1068,27 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi return archivePath, dropPath } +func removePythonWheels(matches []string, version string) []string { + if hasSnapshotEnv() { + version = fmt.Sprintf("%s-SNAPSHOT", version) + } + + var wheels []string + for _, spec := range manifest.ExpectedBinaries { + if spec.PythonWheel { + wheels = append(wheels, spec.GetPackageName(version, "")) + } + } + + cleaned := make([]string, 0, len(matches)) + for _, path := range matches { + if !slices.Contains(wheels, filepath.Base(path)) { + cleaned = append(cleaned, path) + } + } + return cleaned +} + // flattenDependencies will extract all the required packages collected in archivePath and dropPath in flatPath and // regenerate checksums func flattenDependencies(requiredPackages []string, packageVersion, archivePath, dropPath, flatPath string, manifestResponse *manifest.Build) { @@ -1100,6 +1112,10 @@ func flattenDependencies(requiredPackages []string, packageVersion, archivePath, } matches = append(matches, zipMatches...) + // never flatten any python wheels, the packages.yml and docker should handle + // those specifically so that the python wheels are installed into the container + matches = removePythonWheels(matches, packageVersion) + if mg.Verbose() { log.Printf("--- Extracting into the flat dir: %v", matches) } diff --git a/specs/connectors.spec.yml b/specs/connectors.spec.yml new file mode 100644 index 00000000000..0fb61f0bba2 --- /dev/null +++ b/specs/connectors.spec.yml @@ -0,0 +1,17 @@ +version: 2 +inputs: + - name: connectors-py + description: "Connectors Python" + platforms: + - linux/amd64 + - linux/arm64 + - container/amd64 + - container/arm64 + outputs: + - elasticsearch + command: + restart_monitoring_period: 5s + maximum_restarts_per_period: 1 + timeouts: + restart: 1s + args: [] From f9b3a8c4cbaff0b5cc16696bbdb1075707cb018e Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Fri, 6 Sep 2024 17:18:57 -0400 Subject: [PATCH 3/9] Call install-agent. --- dev-tools/packaging/packages.yml | 2 +- .../templates/docker/Dockerfile.elastic-agent.tmpl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 24fcaee3fcf..ef6425989f4 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -295,7 +295,7 @@ shared: - &agent_docker_service_spec docker_variant: 'service' files: - 'data/service_downloads/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip': + 'data/service/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip': source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip' mode: 0755 'data/{{.BeatName}}-{{ commit_short }}/components/connectors': diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 967a331d003..59b12393af0 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -178,9 +178,9 @@ RUN mkdir /app && \ {{- end }} {{- if eq .Variant "service" }} -RUN apk add --no-cache python-3.12 py3.12-pip && \ - pip install {{ $beatHome }}/data/service_downloads/connectors-*.zip && \ - rm -rf {{ $beatHome }}/data/service_downloads && \ +RUN apk add --no-cache git make python-3.11 py3.11-pip && \ + unzip {{ $beatHome }}/data/service/connectors-*.zip -d {{ $beatHome }}/data/service && \ + make -C {{ $beatHome }}/data/service/elasticsearch_connectors-* install-agent && \ chmod 0755 {{ $beatHome }}/data/elastic-agent-*/components/connectors {{- end }} From 6794fe0daf630685bf80e767edd730a6d98a821b Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 12 Sep 2024 11:26:13 -0400 Subject: [PATCH 4/9] It runs. --- dev-tools/packaging/files/linux/connectors.py | 10 ---------- dev-tools/packaging/files/linux/connectors.sh | 6 ++++++ dev-tools/packaging/packages.yml | 2 +- .../templates/docker/Dockerfile.elastic-agent.tmpl | 3 ++- 4 files changed, 9 insertions(+), 12 deletions(-) delete mode 100755 dev-tools/packaging/files/linux/connectors.py create mode 100755 dev-tools/packaging/files/linux/connectors.sh diff --git a/dev-tools/packaging/files/linux/connectors.py b/dev-tools/packaging/files/linux/connectors.py deleted file mode 100755 index 7f9dcbdc552..00000000000 --- a/dev-tools/packaging/files/linux/connectors.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -import time - -def main(): - while True: - time.sleep(1) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/dev-tools/packaging/files/linux/connectors.sh b/dev-tools/packaging/files/linux/connectors.sh new file mode 100755 index 00000000000..81f95e24372 --- /dev/null +++ b/dev-tools/packaging/files/linux/connectors.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +PY_AGENT_CLIENT_PATH=/usr/share/connectors +PYTHON_PATH=$PY_AGENT_CLIENT_PATH/.venv/bin/python +COMPONENT_PATH=$PY_AGENT_CLIENT_PATH/connectors/agent/cli.py +$PYTHON_PATH $COMPONENT_PATH diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index ef6425989f4..369c4707d89 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -299,7 +299,7 @@ shared: source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip' mode: 0755 'data/{{.BeatName}}-{{ commit_short }}/components/connectors': - source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/connectors.py' + source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/connectors.sh' mode: 0755 'data/{{.BeatName}}-{{ commit_short }}/components/connectors.spec.yml': source: '{{ elastic_beats_dir }}/specs/connectors.spec.yml' diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 59b12393af0..7d6a6a65388 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -180,7 +180,8 @@ RUN mkdir /app && \ {{- if eq .Variant "service" }} RUN apk add --no-cache git make python-3.11 py3.11-pip && \ unzip {{ $beatHome }}/data/service/connectors-*.zip -d {{ $beatHome }}/data/service && \ - make -C {{ $beatHome }}/data/service/elasticsearch_connectors-* install-agent && \ + mv {{ $beatHome }}/data/service/elasticsearch_connectors-* /usr/share/connectors && \ + PYTHON=python3.11 make -C /usr/share/connectors clean install install-agent && \ chmod 0755 {{ $beatHome }}/data/elastic-agent-*/components/connectors {{- end }} From 3a002f16a2dd58d22c06a8abdd6ee23efd97aa10 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 12 Sep 2024 12:30:35 -0400 Subject: [PATCH 5/9] Fix lint. --- dev-tools/mage/dockervariants.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dev-tools/mage/dockervariants.go b/dev-tools/mage/dockervariants.go index 2e0ec389926..66acbf66a7e 100644 --- a/dev-tools/mage/dockervariants.go +++ b/dev-tools/mage/dockervariants.go @@ -10,14 +10,14 @@ import ( ) const ( - undefined = "undefined" - basic = "basic" - ubi = "ubi" - wolfi = "wolfi" - complete = "complete" + undefined = "undefined" + basic = "basic" + ubi = "ubi" + wolfi = "wolfi" + complete = "complete" wolfiComplete = "wolfi-complete" - cloud = "cloud" - service = "service" + cloud = "cloud" + service = "service" ) // DockerVariant defines the docker variant to build. From 4e465da064737db4965f58c47d9e1d25f305953d Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Fri, 20 Sep 2024 16:29:39 -0400 Subject: [PATCH 6/9] Add integration test. --- pkg/testing/kubernetes/supported.go | 4 + .../kubernetes_agent_service_test.go | 129 ++++++++++++++++++ .../kubernetes_agent_standalone_test.go | 83 +++++++---- .../integration/testdata/connectors.agent.yml | 13 ++ 4 files changed, 200 insertions(+), 29 deletions(-) create mode 100644 testing/integration/kubernetes_agent_service_test.go create mode 100644 testing/integration/testdata/connectors.agent.yml diff --git a/pkg/testing/kubernetes/supported.go b/pkg/testing/kubernetes/supported.go index e8d8f96cf1c..a5545904d72 100644 --- a/pkg/testing/kubernetes/supported.go +++ b/pkg/testing/kubernetes/supported.go @@ -67,6 +67,10 @@ var variants = []struct { Name: "cloud", Image: "docker.elastic.co/beats-ci/elastic-agent-cloud", }, + { + Name: "service", + Image: "docker.elastic.co/beats-ci/elastic-agent-service", + }, } // GetSupported returns the list of supported OS types for Kubernetes. diff --git a/testing/integration/kubernetes_agent_service_test.go b/testing/integration/kubernetes_agent_service_test.go new file mode 100644 index 00000000000..4a5ebdda2ad --- /dev/null +++ b/testing/integration/kubernetes_agent_service_test.go @@ -0,0 +1,129 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build integration + +package integration + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + + "github.com/elastic/elastic-agent/pkg/testing/define" +) + +func TestKubernetesAgentService(t *testing.T) { + info := define.Require(t, define.Requirements{ + Stack: &define.Stack{}, + Local: false, + Sudo: false, + OS: []define.OS{ + // only test the service container + {Type: define.Kubernetes, DockerVariant: "service"}, + }, + Group: define.Kubernetes, + }) + + agentImage := os.Getenv("AGENT_IMAGE") + require.NotEmpty(t, agentImage, "AGENT_IMAGE must be set") + + client, err := info.KubeClient() + require.NoError(t, err) + require.NotNil(t, client) + + testLogsBasePath := os.Getenv("K8S_TESTS_POD_LOGS_BASE") + require.NotEmpty(t, testLogsBasePath, "K8S_TESTS_POD_LOGS_BASE must be set") + + err = os.MkdirAll(filepath.Join(testLogsBasePath, t.Name()), 0755) + require.NoError(t, err, "failed to create test logs directory") + + namespace := info.Namespace + + esHost := os.Getenv("ELASTICSEARCH_HOST") + require.NotEmpty(t, esHost, "ELASTICSEARCH_HOST must be set") + + esAPIKey, err := generateESAPIKey(info.ESClient, namespace) + require.NoError(t, err, "failed to generate ES API key") + require.NotEmpty(t, esAPIKey, "failed to generate ES API key") + + renderedManifest, err := renderKustomize(agentK8SKustomize) + require.NoError(t, err, "failed to render kustomize") + + hasher := sha256.New() + hasher.Write([]byte(t.Name())) + testNamespace := strings.ToLower(base64.URLEncoding.EncodeToString(hasher.Sum(nil))) + testNamespace = noSpecialCharsRegexp.ReplaceAllString(testNamespace, "") + + k8sObjects, err := yamlToK8SObjects(bufio.NewReader(bytes.NewReader(renderedManifest))) + require.NoError(t, err, "failed to convert yaml to k8s objects") + + adjustK8SAgentManifests(k8sObjects, testNamespace, "elastic-agent-standalone", + func(container *corev1.Container) { + // set agent image + container.Image = agentImage + // set ImagePullPolicy to "Never" to avoid pulling the image + // as the image is already loaded by the kubernetes provisioner + container.ImagePullPolicy = "Never" + + // set Elasticsearch host and API key + for idx, env := range container.Env { + if env.Name == "ES_HOST" { + container.Env[idx].Value = esHost + container.Env[idx].ValueFrom = nil + } + if env.Name == "API_KEY" { + container.Env[idx].Value = esAPIKey + container.Env[idx].ValueFrom = nil + } + } + + // has a unique entrypoint and command because its ran in the cloud + // adjust the spec to run it correctly + container.Command = []string{"elastic-agent"} + container.Args = []string{"-c", "/etc/elastic-agent/agent.yml", "-e"} + }, + func(pod *corev1.PodSpec) { + for volumeIdx, volume := range pod.Volumes { + // need to update the volume path of the state directory + // to match the test namespace + if volume.Name == "elastic-agent-state" { + hostPathType := corev1.HostPathDirectoryOrCreate + pod.Volumes[volumeIdx].VolumeSource.HostPath = &corev1.HostPathVolumeSource{ + Type: &hostPathType, + Path: fmt.Sprintf("/var/lib/elastic-agent-standalone/%s/state", testNamespace), + } + } + } + }) + + // update the configmap to only run the connectors input + serviceAgentYAML, err := os.ReadFile(filepath.Join("testdata", "connectors.agent.yml")) + require.NoError(t, err) + for _, obj := range k8sObjects { + switch objWithType := obj.(type) { + case *corev1.ConfigMap: + _, ok := objWithType.Data["agent.yml"] + if ok { + objWithType.Data["agent.yml"] = string(serviceAgentYAML) + } + } + } + + ctx := context.Background() + + deployK8SAgent(t, ctx, client, k8sObjects, testNamespace, false, testLogsBasePath, map[string]bool{ + "connectors-py": true, + }) +} diff --git a/testing/integration/kubernetes_agent_standalone_test.go b/testing/integration/kubernetes_agent_standalone_test.go index 417a36c30c3..6ad0de332c3 100644 --- a/testing/integration/kubernetes_agent_standalone_test.go +++ b/testing/integration/kubernetes_agent_standalone_test.go @@ -15,6 +15,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" @@ -23,11 +24,8 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - - "github.com/elastic/elastic-agent/pkg/testing/define" - "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" "github.com/elastic/go-elasticsearch/v8" + "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" @@ -47,6 +45,11 @@ import ( "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" + + aclient "github.com/elastic/elastic-agent/pkg/control/v2/client" + atesting "github.com/elastic/elastic-agent/pkg/testing" + "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" ) const ( @@ -657,46 +660,68 @@ func deployK8SAgent(t *testing.T, ctx context.Context, client klient.Client, obj require.NotEmpty(t, agentPodName, "agent pod name is empty") - command := []string{"elastic-agent", "status"} + command := []string{"elastic-agent", "status", "--output=json"} + var status atesting.AgentStatusOutput var stdout, stderr bytes.Buffer var agentHealthyErr error // we will wait maximum 120 seconds for the agent to report healthy for i := 0; i < 120; i++ { + status = atesting.AgentStatusOutput{} // clear status output stdout.Reset() stderr.Reset() agentHealthyErr = client.Resources().ExecInPod(ctx, namespace, agentPodName, "elastic-agent-standalone", command, &stdout, &stderr) if agentHealthyErr == nil { - break + if uerr := json.Unmarshal(stdout.Bytes(), &status); uerr == nil { + if status.State == int(aclient.Healthy) { + // agent is healthy innner tests should now pass + if runInnerK8STests { + err := client.Resources().ExecInPod(ctx, namespace, agentPodName, "elastic-agent-standalone", + []string{"/usr/share/elastic-agent/k8s-inner-tests", "-test.v"}, &stdout, &stderr) + t.Log(stdout.String()) + if err != nil { + t.Log(stderr.String()) + } + require.NoError(t, err, "error at k8s inner tests execution") + } + + // validate that the components defined are also healthy if they should exist + componentsCorrect := true + for component, shouldBePresent := range componentPresence { + compState, ok := getComponentState(status, component) + if shouldBePresent { + if !ok { + // doesn't exist + componentsCorrect = false + } else if compState != int(aclient.Healthy) { + // not healthy + componentsCorrect = false + } + } else if ok { + // should not be present + // break instantly and fail (as it existing should never happen) + break + } + } + if componentsCorrect { + // agent health and components are correct + return + } + } + } } time.Sleep(time.Second * 1) } - statusString := stdout.String() - if agentHealthyErr != nil { - t.Errorf("elastic-agent never reported healthy: %v", agentHealthyErr) - t.Logf("stdout: %s\n", statusString) - t.Logf("stderr: %s\n", stderr.String()) - t.FailNow() - return - } - - stdout.Reset() - stderr.Reset() - - for component, shouldBePresent := range componentPresence { - isPresent := strings.Contains(statusString, component) - require.Equal(t, shouldBePresent, isPresent) - } + t.Fatalf("elastic-agent never reported healthy: %+v", status) +} - if runInnerK8STests { - err := client.Resources().ExecInPod(ctx, namespace, agentPodName, "elastic-agent-standalone", - []string{"/usr/share/elastic-agent/k8s-inner-tests", "-test.v"}, &stdout, &stderr) - t.Log(stdout.String()) - if err != nil { - t.Log(stderr.String()) +func getComponentState(status atesting.AgentStatusOutput, componentName string) (int, bool) { + for _, comp := range status.Components { + if comp.Name == componentName { + return comp.State, true } - require.NoError(t, err, "error at k8s inner tests execution") } + return -1, false } // dumpLogs dumps the logs of all pods in the given namespace to the given target directory diff --git a/testing/integration/testdata/connectors.agent.yml b/testing/integration/testdata/connectors.agent.yml new file mode 100644 index 00000000000..5c3466ae8ae --- /dev/null +++ b/testing/integration/testdata/connectors.agent.yml @@ -0,0 +1,13 @@ +outputs: + default: + type: elasticsearch + hosts: + - >- + ${ES_HOST} + api_key: ${API_KEY} +agent: + monitoring: + enabled: false +inputs: + - id: connectors + type: connectors-py From ec872f9e991e142034205ad4230edf183a12c6d8 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Fri, 20 Sep 2024 20:14:31 -0400 Subject: [PATCH 7/9] Add backin output of status command. --- testing/integration/kubernetes_agent_standalone_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testing/integration/kubernetes_agent_standalone_test.go b/testing/integration/kubernetes_agent_standalone_test.go index 6ad0de332c3..fb6700b8910 100644 --- a/testing/integration/kubernetes_agent_standalone_test.go +++ b/testing/integration/kubernetes_agent_standalone_test.go @@ -712,7 +712,10 @@ func deployK8SAgent(t *testing.T, ctx context.Context, client klient.Client, obj time.Sleep(time.Second * 1) } - t.Fatalf("elastic-agent never reported healthy: %+v", status) + t.Errorf("elastic-agent never reported healthy: %+v", status) + t.Logf("stdout: %s\n", stdout.String()) + t.Logf("stderr: %s\n", stderr.String()) + t.FailNow() } func getComponentState(status atesting.AgentStatusOutput, componentName string) (int, bool) { From 99757d316a70ff0cd8dc9359cf02e75da40a3783 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Sat, 21 Sep 2024 10:02:18 -0400 Subject: [PATCH 8/9] mage check --- testing/integration/kubernetes_agent_standalone_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/integration/kubernetes_agent_standalone_test.go b/testing/integration/kubernetes_agent_standalone_test.go index fb6700b8910..ddcbb559cca 100644 --- a/testing/integration/kubernetes_agent_standalone_test.go +++ b/testing/integration/kubernetes_agent_standalone_test.go @@ -24,9 +24,10 @@ import ( "testing" "time" - "github.com/elastic/go-elasticsearch/v8" "github.com/stretchr/testify/require" + "github.com/elastic/go-elasticsearch/v8" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" From 45ef9d4304a70e77d50dfe8341bdfe77d9906a90 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 30 Sep 2024 15:08:33 -0400 Subject: [PATCH 9/9] Update the size of kubernetes runner. --- .buildkite/integration.pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index cccd2ca1908..a11992264e5 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -116,7 +116,9 @@ steps: - "build/diagnostics/*" agents: provider: "gcp" + machineType: "c2-standard-16" image: "family/core-ubuntu-2204" + diskSizeGb: 400 notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests"