diff --git a/.ci/build-docker-images.groovy b/.ci/build-docker-images.groovy index a59aeb8da8f8..d94ad9c79a4f 100644 --- a/.ci/build-docker-images.groovy +++ b/.ci/build-docker-images.groovy @@ -53,13 +53,14 @@ pipeline { expression { return params.RELEASE_TEST_IMAGES } } steps { - dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}") - withMageEnv(){ - dir("${BASE_DIR}/metricbeat"){ - retryWithSleep(retries: 3, seconds: 5, backoff: true){ - sh(label: 'Build', script: "mage compose:buildSupportedVersions"); - sh(label: 'Push', script: "mage compose:pushSupportedVersions"); - } + dir("${BASE_DIR}"){ + script { + def parallelTasks = [:] + + parallelTasks["arm_metricbeat"] = generateBuildStep(platform: "arm", beatPath: 'metricbeat') + parallelTasks["ubuntu_20_metricbeat"] = generateBuildStep(platform: "ubuntu-20", beatPath: 'metricbeat') + + parallel(parallelTasks) } } } @@ -72,13 +73,14 @@ pipeline { expression { return params.RELEASE_TEST_IMAGES } } steps { - dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}") - withMageEnv(){ - dir("${BASE_DIR}/x-pack/metricbeat"){ - retryWithSleep(retries: 3, seconds: 5, backoff: true){ - sh(label: 'Build', script: "mage compose:buildSupportedVersions"); - sh(label: 'Push', script: "mage compose:pushSupportedVersions"); - } + dir("${BASE_DIR}"){ + script { + def parallelTasks = [:] + + parallelTasks["arm_xpack_metricbeat"] = generateBuildStep(platform: "arm", beatPath: 'x-pack/metricbeat') + parallelTasks["ubuntu_20_xpack_metricbeat"] = generateBuildStep(platform: "ubuntu-20", beatPath: 'x-pack/metricbeat') + + parallel(parallelTasks) } } } @@ -91,13 +93,14 @@ pipeline { expression { return params.RELEASE_TEST_IMAGES } } steps { - dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}") - withMageEnv(){ - dir("${BASE_DIR}/x-pack/filebeat"){ - retryWithSleep(retries: 3, seconds: 5, backoff: true){ - sh(label: 'Build', script: "mage compose:buildSupportedVersions"); - sh(label: 'Push', script: "mage compose:pushSupportedVersions"); - } + dir("${BASE_DIR}"){ + script { + def parallelTasks = [:] + + parallelTasks["arm_xpack_filebeat"] = generateBuildStep(platform: "arm", beatPath: 'x-pack/filebeat') + parallelTasks["ubuntu_20_xpack_filebeat"] = generateBuildStep(platform: "ubuntu-20", beatPath: 'x-pack/filebeat') + + parallel(parallelTasks) } } } @@ -109,3 +112,22 @@ pipeline { } } } + +def generateBuildStep(Map args = [:]) { + def beatPath = args.get('beatPath') + def platform = args.get('platform')?.trim() ? args.get('platform') : 'ubuntu-20' + + return { + withNode(labels: "${platform} && immutable && docker", sleepMax: 20, forceWorkspace: true){ + dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}") + withMageEnv(){ + dir("${BASE_DIR}/${beatPath}"){ + retryWithSleep(retries: 3, seconds: 5, backoff: true){ + sh(label: 'Build', script: "mage compose:buildSupportedVersions"); + sh(label: 'Push', script: "mage manifesttool:pushSupportedVersions"); + } + } + } + } + } +} \ No newline at end of file diff --git a/dev-tools/mage/target/manifesttool/manifesttool.go b/dev-tools/mage/target/manifesttool/manifesttool.go new file mode 100644 index 000000000000..67ed08a7bf29 --- /dev/null +++ b/dev-tools/mage/target/manifesttool/manifesttool.go @@ -0,0 +1,168 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package manifesttool + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" + + devtools "github.com/elastic/beats/v7/dev-tools/mage" +) + +const manifestToolImage = "docker.elastic.co/infra/manifest-tool:latest" + +// SupportedVersions is the definition of supported version files +type SupportedVersions struct { + Variants []map[string]string `yaml:"variants"` +} + +// ManifestTool are targets to build multi-platform images with manifest-tool +type ManifestTool mg.Namespace + +// PushSupportedVersions pushes images for versions defined in supported-versions.yml files +func (m ManifestTool) PushSupportedVersions() error { + if runtime.GOOS != "linux" { + return errors.Errorf("pushing supported versions in '%s' is not supported. Only linux is supported at this moment", runtime.GOOS) + } + + fmt.Println(">> manifest-tool: Pushing docker images for supported versions") + return m.pushForEachVariant() +} + +func (m ManifestTool) pushForEachVariant() error { + files, err := findSupportedVersionsFiles() + if err != nil { + return errors.Wrap(err, "finding supported versions files") + } + + for _, f := range files { + err := forEachSupportedVersion(f) + if err != nil { + return errors.Wrapf(err, "pushing supported versions defined in %s", f) + } + } + + return nil +} + +func findSupportedVersionsFiles() ([]string, error) { + if f := os.Getenv("SUPPORTED_VERSIONS_FILE"); len(f) > 0 { + return []string{f}, nil + } + + if module := os.Getenv("MODULE"); len(module) > 0 { + path := filepath.Join("module", module, "_meta/supported-versions.yml") + return []string{path}, nil + } + + if input := os.Getenv("INPUT"); len(input) > 0 { + path := filepath.Join("input", input, "_meta/supported-versions.yml") + return []string{path}, nil + } + + return devtools.FindFilesRecursive(func(path string, _ os.FileInfo) bool { + return filepath.Base(path) == "supported-versions.yml" + }) +} + +func forEachSupportedVersion(file string) error { + d, err := ioutil.ReadFile(file) + if err != nil { + return errors.Wrapf(err, "reading supported versions file %s", file) + } + + module := filepath.Base(filepath.Dir(filepath.Dir(file))) + + moduleUppercase := strings.ToUpper(module) + + var supportedVersions SupportedVersions + + err = yaml.Unmarshal(d, &supportedVersions) + if err != nil { + return errors.Wrapf(err, "parsing supported versions file %s", file) + } + + for _, variant := range supportedVersions.Variants { + index := "1" + var codename string + var version string + for k, v := range variant { + switch k { + case moduleUppercase + "_VERSION": + version = v + case moduleUppercase + "_CODENAME": + case moduleUppercase + "_VARIANT": + codename = v + case moduleUppercase + "_INDEX": + index = v + default: + codename = "" + version = "" + index = "1" + } + } + + var tag string + if codename == "" { + tag = fmt.Sprintf("%s-%s", version, index) + } else { + tag = fmt.Sprintf("%s-%s-%s", codename, version, index) + } + + // supported platforms on CI: linux/amd64, linux/arm64 + platform := runtime.GOOS + "/" + runtime.GOARCH + fmt.Printf(">> manifest-tool: Pushing images for module '%s', tag '%s' on platform '%s'\n", module, tag, platform) + + homeDir, err := os.UserHomeDir() + if err != nil { + return err + } + + // this file path uses *NIX separator, because the images are supposed to be built under linux + dockerConfigDir := homeDir + "/.docker" + + image := fmt.Sprintf("docker.elastic.co/integrations-ci/beats-%s:%s", module, tag) + var stderr bytes.Buffer + _, err = sh.Exec( + map[string]string{}, nil, &stderr, + "docker", "run", "--rm", "--mount", "src="+dockerConfigDir+",target=/docker-config,type=bind", + manifestToolImage, + "--platforms", platform, + "--template", image, + "--target", image, + ) + if err != nil { + io.Copy(os.Stderr, &stderr) + return err + } + } + fmt.Println(">> manifest-tool: OK") + + return nil +} diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index 661239410b21..845fbe7860e3 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -51,6 +51,8 @@ import ( // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/compose" // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/manifesttool" + // mage:import _ "github.com/elastic/beats/v7/metricbeat/scripts/mage/target/metricset" ) diff --git a/metricbeat/module/ceph/_meta/supported-versions.yml b/metricbeat/module/ceph/_meta/supported-versions.yml index b8fc1d529c5b..eeded4ec9dfc 100644 --- a/metricbeat/module/ceph/_meta/supported-versions.yml +++ b/metricbeat/module/ceph/_meta/supported-versions.yml @@ -1,5 +1,10 @@ variants: - CEPH_CODENAME: jewel CEPH_VERSION: master-6373c6a-jewel-centos-7-x86_64 + CEPH_INDEX: 1 + - CEPH_CODENAME: jewel + CEPH_VERSION: master-6373c6a-jewel-centos-7-x86_64 + CEPH_INDEX: 2 - CEPH_CODENAME: nautilus CEPH_VERSION: master-97985eb-nautilus-centos-7-x86_64 + CEPH_INDEX: 2 diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index 9c7f436e2e43..a5341feace92 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -24,6 +24,8 @@ import ( // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/compose" // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/manifesttool" + // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/test" diff --git a/x-pack/metricbeat/magefile.go b/x-pack/metricbeat/magefile.go index 0771209d0972..55e8d0c781fb 100644 --- a/x-pack/metricbeat/magefile.go +++ b/x-pack/metricbeat/magefile.go @@ -26,6 +26,8 @@ import ( // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/compose" // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/manifesttool" + // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/test" // mage:import _ "github.com/elastic/beats/v7/metricbeat/scripts/mage/target/metricset" diff --git a/x-pack/packetbeat/magefile.go b/x-pack/packetbeat/magefile.go index 0cdeba15dcd3..cb1aeffc4a56 100644 --- a/x-pack/packetbeat/magefile.go +++ b/x-pack/packetbeat/magefile.go @@ -21,6 +21,8 @@ import ( // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/compose" // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/manifesttool" + // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test"