diff --git a/.ci/build-docker-images.groovy b/.ci/build-docker-images.groovy new file mode 100644 index 0000000000..b93fd619b0 --- /dev/null +++ b/.ci/build-docker-images.groovy @@ -0,0 +1,94 @@ +#!/usr/bin/env groovy + +@Library('apm@current') _ + +pipeline { + agent { label 'ubuntu-20' } + environment { + REPO = 'e2e-testing' + BASE_DIR = "src/github.com/elastic/${env.REPO}" + DOCKER_REGISTRY = 'docker.elastic.co' + DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod' + HOME = "${env.WORKSPACE}" + NOTIFY_TO = credentials('notify-to') + PIPELINE_LOG_LEVEL = 'INFO' + JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" + } + options { + timeout(time: 1, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20')) + timestamps() + ansiColor('xterm') + disableResume() + durabilityHint('PERFORMANCE_OPTIMIZED') + rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) + quietPeriod(10) + } + triggers { + cron 'H H(0-5) * * 1-5' + } + stages { + stage('Checkout') { + steps { + deleteDir() + gitCheckout(basedir: "${BASE_DIR}", + branch: "${params.BRANCH_REFERENCE}", + repo: "https://github.com/elastic/${REPO}.git", + credentialsId: "${JOB_GIT_CREDENTIALS}" + ) + stash allowEmpty: true, name: 'source', useDefaultExcludes: false + } + } + stage('Build AMD Docker images'){ + agent { label 'ubuntu-20 && immutable && docker' } + environment { + HOME = "${env.WORKSPACE}/${BASE_DIR}" + } + steps { + deleteDir() + unstash 'source' + dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + dir("${BASE_DIR}") { + withEnv(["ARCH=amd64"]) { + sh(label: 'Build AMD images', script: '.ci/scripts/build-docker-images.sh') + } + } + } + } + stage('Build ARM Docker images'){ + agent { label 'arm && immutable && docker' } + environment { + HOME = "${env.WORKSPACE}/${BASE_DIR}" + } + steps { + deleteDir() + unstash 'source' + dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + dir("${BASE_DIR}") { + withEnv(["ARCH=arm64"]) { + sh(label: 'Build ARM images', script: '.ci/scripts/build-docker-images.sh') + } + } + } + } + stage('Push multiplatform manifest'){ + agent { label 'ubuntu-20 && immutable && docker' } + environment { + HOME = "${env.WORKSPACE}/${BASE_DIR}" + } + steps { + deleteDir() + unstash 'source' + dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + dir("${BASE_DIR}") { + sh(label: 'Push multiplatform manifest', script: '.ci/scripts/push-multiplatform-manifest.sh') + } + } + } + } + post { + cleanup { + notifyBuildResult() + } + } +} diff --git a/.ci/docker/centos-systemd/Dockerfile b/.ci/docker/centos-systemd/Dockerfile new file mode 100644 index 0000000000..4b0bc4b483 --- /dev/null +++ b/.ci/docker/centos-systemd/Dockerfile @@ -0,0 +1,18 @@ +FROM centos:7 + +ENV container docker + +LABEL maintainer="manuel.delapena@elastic.co" + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ +rm -f /lib/systemd/system/multi-user.target.wants/*;\ +rm -f /etc/systemd/system/*.wants/*;\ +rm -f /lib/systemd/system/local-fs.target.wants/*; \ +rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ +rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ +rm -f /lib/systemd/system/basic.target.wants/*;\ +rm -f /lib/systemd/system/anaconda.target.wants/*; + +VOLUME [ "/sys/fs/cgroup" ] + +CMD ["/usr/sbin/init"] diff --git a/.ci/docker/debian-systemd/Dockerfile b/.ci/docker/debian-systemd/Dockerfile new file mode 100644 index 0000000000..a503483689 --- /dev/null +++ b/.ci/docker/debian-systemd/Dockerfile @@ -0,0 +1,141 @@ +# This file is part of docker-debian-systemd. +# +# Copyright (c) +# 2018-2019 Alexander Haase +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# This image bases on the regular Debian image. By default the 'latest' tag +# (pointing to the current stable release) of the parent image will be used. +# However, an alternate parent tag may be set by defining the 'TAG' build +# argument to a specific Debian release, e.g. 'stretch' or 'buster'. +ARG TAG=latest +FROM debian:${TAG} +LABEL maintainer="manuel.delapena@elastic.co" + +# Configure the debconf frontend. +# +# This image doesn't include whiptail, dialog, nor the readline perl module. +# Therefore, the debconf frontend will be set to 'teletype' to avoid error +# messages about no dialog frontend could be found. +RUN echo 'debconf debconf/frontend select teletype' | debconf-set-selections + + +# Install the necessary packages. +# +# In addition to the regular Debian base image, a BASIC set of packages from the +# Debian minimal configuration will be installed. After all packages have been +# installed, the apt caches and some log files will be removed to minimize the +# image. +# +# NOTE: An upgrade will be performed to include updates and security fixes of +# installed packages that received updates in the Debian repository after +# the upstream image has been created. +# +# NOTE: No syslog daemon will be installed, as systemd's journald should fit +# most needs. Please file an issue if you think this should be changed. +RUN apt-get update +RUN apt-get dist-upgrade -y +RUN apt-get install -y --no-install-recommends \ + systemd \ + systemd-sysv \ + cron \ + anacron + +RUN apt-get clean +RUN rm -rf \ + /var/lib/apt/lists/* \ + /var/log/alternatives.log \ + /var/log/apt/history.log \ + /var/log/apt/term.log \ + /var/log/dpkg.log + + +# Configure systemd. +# +# For running systemd inside a Docker container, some additional tweaks are +# required. For a detailed list see: +# +# https://developers.redhat.com/blog/2016/09/13/ \ +# running-systemd-in-a-non-privileged-container/ +# +# Additional tweaks will be applied in the final image below. + +# To avoid ugly warnings when running this image on a host running systemd, the +# following units will be masked. +# +# NOTE: This will not remove ALL warnings in all Debian releases, but seems to +# work for stretch. +RUN systemctl mask -- \ + dev-hugepages.mount \ + sys-fs-fuse-connections.mount + +# The machine-id should be generated when creating the container. This will be +# done automatically if the file is not present, so let's delete it. +RUN rm -f \ + /etc/machine-id \ + /var/lib/dbus/machine-id + + + + +# Build the final image. +# +# To get a minimal image without deleted files in intermediate layers, the +# contents of the image previously built will be copied into a second version of +# the parent image. +# +# NOTE: This method requires buildkit, as the differ of buildkit will copy +# changed files only and we'll get a minimal image with just the changed +# files in a single new layer. +# +# NOTE: All settings related to the image's environment (e.g. CMD, ENV and +# VOLUME settings) need to be set in the following image definition to be +# used by child images and containers. + +FROM debian:${TAG} +COPY --from=0 / / + + +# Configure systemd. +# +# For running systemd inside a Docker container, some additional tweaks are +# required. Some of them have already been applied above. +# +# The 'container' environment variable tells systemd that it's running inside a +# Docker container environment. +ENV container docker + +# A different stop signal is required, so systemd will initiate a shutdown when +# running 'docker stop '. +STOPSIGNAL SIGRTMIN+3 + +# The host's cgroup filesystem need's to be mounted (read-only) in the +# container. '/run', '/run/lock' and '/tmp' need to be tmpfs filesystems when +# running the container without 'CAP_SYS_ADMIN'. +# +# NOTE: For running Debian stretch, 'CAP_SYS_ADMIN' still needs to be added, as +# stretch's version of systemd is not recent enough. Buster will run just +# fine without 'CAP_SYS_ADMIN'. +VOLUME [ "/sys/fs/cgroup", "/run", "/run/lock", "/tmp" ] + +# As this image should run systemd, the default command will be changed to start +# the init system. CMD will be preferred in favor of ENTRYPOINT, so one may +# override it when creating the container to e.g. to run a bash console instead. +CMD [ "/sbin/init" ] diff --git a/.ci/jobs/build-docker-images.yml b/.ci/jobs/build-docker-images.yml new file mode 100644 index 0000000000..dcadbb2915 --- /dev/null +++ b/.ci/jobs/build-docker-images.yml @@ -0,0 +1,27 @@ +--- +- job: + name: Beats/build-docker-images + display-name: E2E Tests Docker images + description: Job to pre-build docker images used in E2E tests. + view: Beats + project-type: pipeline + parameters: + - string: + name: BRANCH_REFERENCE + default: master + description: the Git branch specifier + pipeline-scm: + script-path: .ci/build-docker-images.groovy + scm: + - git: + url: git@github.com:elastic/e2e-testint.git + refspec: +refs/heads/*:refs/remotes/origin/* + wipe-workspace: true + name: origin + shallow-clone: true + credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba + reference-repo: /var/lib/jenkins/.git-references/e2e-testing.git + branches: + - $BRANCH_REFERENCE + triggers: + - timed: 'H H(0-5) * * 1-5' diff --git a/.ci/scripts/build-docker-images.sh b/.ci/scripts/build-docker-images.sh new file mode 100755 index 0000000000..98d4d09997 --- /dev/null +++ b/.ci/scripts/build-docker-images.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +## Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +## or more contributor license agreements. Licensed under the Elastic License; +## you may not use this file except in compliance with the Elastic License. + +set -euxo pipefail + +ARCH="${ARCH:-amd64}" + +readonly ELASTIC_REGISTRY="docker.elastic.co" +readonly OBSERVABILITY_CI_REGISTRY="${ELASTIC_REGISTRY}/observability-ci" + +main() { + _build_and_push "centos-systemd" + _build_and_push "debian-systemd" +} + +_build_and_push() { + local image="${1}" + + local platformSpecificImage="${OBSERVABILITY_CI_REGISTRY}/${image}-${ARCH}:latest" + + docker build -t ${platformSpecificImage} .ci/docker/${image} + + docker push ${platformSpecificImage} +} + +main "$@" diff --git a/.ci/scripts/push-multiplatform-manifest.sh b/.ci/scripts/push-multiplatform-manifest.sh new file mode 100755 index 0000000000..56f5771b0e --- /dev/null +++ b/.ci/scripts/push-multiplatform-manifest.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +## Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +## or more contributor license agreements. Licensed under the Elastic License; +## you may not use this file except in compliance with the Elastic License. + +set -euxo pipefail + +readonly ELASTIC_REGISTRY="docker.elastic.co" +readonly MANIFEST_TOOL_IMAGE="${ELASTIC_REGISTRY}/infra/manifest-tool:latest" +readonly OBSERVABILITY_CI_REGISTRY="${ELASTIC_REGISTRY}/observability-ci" + +main() { + _push_multiplatform_manifest "centos-systemd" + _push_multiplatform_manifest "debian-systemd" +} + +_push_multiplatform_manifest() { + local image="${1}" + + local fqn="${OBSERVABILITY_CI_REGISTRY}/${image}:latest" + # the '-ARCH' placeholder will be replaced with the values in the '--platforms' argument + local templateFqn="${OBSERVABILITY_CI_REGISTRY}/${image}-ARCH:latest" + + docker run --rm \ + --mount src=${HOME}/.docker,target=/docker-config,type=bind \ + ${MANIFEST_TOOL_IMAGE} --docker-cfg "/docker-config" \ + push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template ${templateFqn} \ + --target ${fqn} +} + +main "$@" diff --git a/cli/config/compose/services/centos-systemd/docker-compose.yml b/cli/config/compose/services/centos-systemd/docker-compose.yml index 1d3b77d01a..6721740dcb 100644 --- a/cli/config/compose/services/centos-systemd/docker-compose.yml +++ b/cli/config/compose/services/centos-systemd/docker-compose.yml @@ -4,6 +4,7 @@ services: image: centos/systemd:${centos_systemdTag:-latest} container_name: ${centos_systemdContainerName} entrypoint: "/usr/sbin/init" + platform: ${stackPlatform:-linux/amd64} privileged: true volumes: - ${centos_systemdAgentBinarySrcPath:-.}:${centos_systemdAgentBinaryTargetPath:-/tmp} diff --git a/cli/config/compose/services/debian-systemd/docker-compose.yml b/cli/config/compose/services/debian-systemd/docker-compose.yml index b6d15cd1b1..ae47b83aac 100644 --- a/cli/config/compose/services/debian-systemd/docker-compose.yml +++ b/cli/config/compose/services/debian-systemd/docker-compose.yml @@ -4,6 +4,7 @@ services: image: alehaa/debian-systemd:${debian_systemdTag:-stretch} container_name: ${debian_systemdContainerName} entrypoint: "/sbin/init" + platform: ${stackPlatform:-linux/amd64} privileged: true volumes: - ${debian_systemdAgentBinarySrcPath:-.}:${debian_systemdAgentBinaryTargetPath:-/tmp}