diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile deleted file mode 100644 index 23f3ca1e707..00000000000 --- a/.ci/Jenkinsfile +++ /dev/null @@ -1,316 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -pipeline { - agent { label 'ubuntu-22 && immutable' } - environment { - REPO = "elastic-agent" - BASE_DIR = "src/github.com/elastic/${env.REPO}" - JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" - PIPELINE_LOG_LEVEL = 'INFO' - SNAPSHOT = true - JOB_GCS_CREDENTIALS = 'fleet-ci-gcs-plugin' // Support stash/unstash v2 - JOB_GCS_BUCKET = 'fleet-ci-temp' // Support stash/unstash v2 - JOB_GCS_EXT_BUCKET = 'fleet-ci-artifacts' // Support uploadPackagesToGoogleBucket - JOB_GCS_EXT_CREDENTIALS = 'fleet-ci-gcs-plugin-file-credentials' // Support uploadPackagesToGoogleBucket - DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' - DOCKER_REGISTRY = 'docker.elastic.co' - DEVELOPER_MODE=true - } - options { - timeout(time: 3, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) - quietPeriod(10) - } - triggers { - issueCommentTrigger("(${obltGitHubComments()}|^run (integration|end-to-end) tests|/package)") - } - parameters { - // disabled by default, but required for merge, there are two GH checks: - // opt-in with 'ci:integration' - booleanParam(name: 'integration_tests_ci', defaultValue: false, description: 'Enable Integration tests') - - // disabled by default, but required for merge: - // opt-in with 'ci:end-to-end' tag on PR - booleanParam(name: 'end_to_end_tests_ci', defaultValue: false, description: 'Enable End-to-End tests') - - // disabled by default, but required for merge: - // opt-in with 'ci:extended-windows' tag on PR - booleanParam(name: 'extended_windows_ci', defaultValue: false, description: 'Enable Extended Windows tests') - - // disabled by default, but required for merge: - // opt-in with 'ci:extended-m1' tag on PR - booleanParam(name: 'extended_m1_ci', defaultValue: false, description: 'Enable M1 tests') - } - stages { - stage('Checkout') { - steps { - pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) - deleteDir() - gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) - stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true).toString()) - setEnvVar('PACKAGING_CHANGES', isGitRegionMatch(patterns: [ '(^dev-tools/packaging/.*|.ci/Jenkinsfile|.go-version|Dockerfile)' ], shouldMatchAll: false).toString()) - setEnvVar('K8S_CHANGES', isGitRegionMatch(patterns: [ '(^deploy/kubernetes/.*|^version/docs/version.asciidoc|.ci/Jenkinsfile)' ], shouldMatchAll: false).toString()) - setEnvVar('EXT_WINDOWS_CHANGES', isGitRegionMatch(patterns: [ '.ci/Jenkinsfile' ], shouldMatchAll: false).toString()) - setEnvVar('EXT_M1_CHANGES', isGitRegionMatch(patterns: [ '.ci/Jenkinsfile' ], shouldMatchAll: false).toString()) - // set the GO_VERSION env variable with the go version to be used in withMageEnv - setEnvVar('GO_VERSION', readFile(file: '.go-version')?.trim()) - } - } - } - stage('Check'){ - steps { - withGithubNotify(context: "Check") { - withMageEnv(){ - dir("${BASE_DIR}"){ - setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) - log(level: 'INFO', text: "env.BEAT_VERSION=${env.BEAT_VERSION}") - cmd(label: 'check', script: 'make check-ci') - } - } - } - } - } - stage('Test') { - when { - beforeAgent true - expression { return env.ONLY_DOCS == "false" } - } - failFast false - matrix { - agent {label "${PLATFORM}"} - options { skipDefaultCheckout() } - axes { - axis { - name 'PLATFORM' - // Orka workers are not healthy (memory and connectivity issues) - values 'ubuntu-22 && immutable', 'aws && aarch64 && gobld/diskSizeGb:200', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable' //, 'macos12 && x86_64' - } - } - stages { - stage('Package') { - when { - beforeAgent true - allOf { - anyOf { - expression { return isE2eEnabled() } - expression { return isPackageEnabled() } - not { changeRequest() } - } - // Run packaging only for the linux specific arch - expression { return (PLATFORM.contains('ubuntu') || PLATFORM.contains('aarch64')) } - } - } - environment { - ARCH = "${PLATFORM.contains('aarch64') ? 'arm64' : 'amd64'}" - DEV = true - EXTERNAL = true - } - steps { - withGithubNotify(context: "Package ${PLATFORM}") { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - withMageEnv(){ - dir("${BASE_DIR}"){ - withPackageEnv("${PLATFORM}") { - cmd(label: 'Go package', script: 'mage package ironbank') - uploadPackagesToGoogleBucket( - credentialsId: env.JOB_GCS_EXT_CREDENTIALS, - repo: env.REPO, - bucket: env.JOB_GCS_EXT_BUCKET, - pattern: "build/distributions/**/*") - pushDockerImages( - registry: env.DOCKER_REGISTRY, - secret: env.DOCKER_ELASTIC_SECRET, - snapshot: env.SNAPSHOT, - version: env.BEAT_VERSION, - images: [ - [ source: "beats/elastic-agent", arch: env.ARCH, target: "observability-ci/elastic-agent"], - [ source: "beats/elastic-agent-oss", arch: env.ARCH, target: "observability-ci/elastic-agent-oss"], - [ source: "beats/elastic-agent-ubi9", arch: env.ARCH, target: "observability-ci/elastic-agent-ubi9"], - [ source: "beats/elastic-agent-complete", arch: env.ARCH, target: "observability-ci/elastic-agent-complete"], - [ source: "beats-ci/elastic-agent-cloud", arch: env.ARCH, target: "observability-ci/elastic-agent-cloud"] - ] - ) - } - } - } - } - } - } - } - } - } - stage('Sync K8s') { //This stage opens a PR to kibana Repository in order to sync k8s manifests - when { - // Only on main branch - // Enable if k8s related changes. - allOf { - branch 'main' // Only runs for branch main - expression { return env.K8S_CHANGES == "true" } // If k8s changes - } - } - failFast false - agent {label 'ubuntu-22 && immutable'} - options { skipDefaultCheckout() } - stages { - stage('OpenKibanaPR') { - steps { - withGhEnv(version: '2.4.0') { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}/deploy/kubernetes"){ - sh(label: '[File Creation] Create-Needed-Manifest', script: """ - WITHOUTCONFIG=true make generate-k8s - ./creator_k8s_manifest.sh . """) - sh(label: '[Clone] Kibana-Repository', script: """ - make ci-clone-kibana-repository - cp Makefile ./kibana - cd kibana - make ci-create-kubernetes-templates-pull-request """) - } - } - } - post { - always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml") - } - } - } - } - } - } - post { - cleanup { - notifyBuildResult(prComment: true, - analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: (isPR() ? env.CHANGE_TARGET : env.BRANCH_NAME)), - githubIssue: false, // Disable creating gh issues for build failures while the E2E tests are stabilized. - githubLabels: 'Team:Elastic-Agent-Control-Plane') - } - } -} - -// As agreed let's report the code coverage for Linux but no ARM only. -def isCodeCoverageEnabled() { - return (isUnix() && !isArm()) -} - -def withPackageEnv(platform, Closure body) { - if (isUnix()) { - if (isDarwin()) { - withPackageDarwinEnv() { - body() - } - } else { - if (isArm()) { - withPackageArmEnv() { - body() - } - } else { - withPackageLinuxEnv() { - body() - } - } - } - } else { - error 'withPackageEnv: unsupported platform' - } -} - -def withPackageLinuxEnv(Closure body) { - // Copied from https://github.com/elastic/beats/blob/e6e65aa92fe355c95789691ebf5a3bcecaf5b4ea/.ci/packaging.groovy#L126-L142 - def PLATFORMS = [ '+all', - 'linux/amd64', - 'linux/386', - 'linux/arm64', - // armv7 packaging isn't working, and we don't currently - // need it for release. Do not re-enable it without - // confirming it is fixed, you will break the packaging - // pipeline! - //'linux/armv7', - // The platforms above are disabled temporarly as crossbuild images are - // not available. See: https://github.com/elastic/golang-crossbuild/issues/71 - //'linux/ppc64le', - //'linux/mips64', - //'linux/s390x', - 'windows/amd64', - 'windows/386' - ].join(' ') - withEnv([ - "PLATFORMS=${PLATFORMS}" - ]) { - body() - } -} - -def withPackageArmEnv(Closure body) { - // Copied from https://github.com/elastic/beats/blob/e6e65aa92fe355c95789691ebf5a3bcecaf5b4ea/.ci/packaging.groovy#L126-L142 - def PLATFORMS = [ 'linux/arm64' ].join(' ') - withEnv([ - "PLATFORMS=${PLATFORMS}", - "PACKAGES=docker" - ]) { - body() - } -} - -def withPackageDarwinEnv(Closure body) { - // Copied from https://github.com/elastic/beats/blob/e6e65aa92fe355c95789691ebf5a3bcecaf5b4ea/.ci/packaging.groovy#L126-L142 - def PLATFORMS = [ '+all', - 'darwin/amd64', - 'darwin/arm64', - ].join(' ') - withEnv([ - "PLATFORMS=${PLATFORMS}" - ]) { - body() - } -} - -def runK8s(Map args=[:]) { - withGithubNotify(context: args.context) { - withMageEnv(){ - withKindEnv(args) { - dir("${BASE_DIR}"){ - sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") - } - } - } - } -} - -/** -* Wrapper to know if the build should enalbe the e2e stage -*/ -def isE2eEnabled() { - return params.end_to_end_tests_ci || env.GITHUB_COMMENT?.contains('e2e tests') || matchesPrLabel(label: 'ci:end-to-end') -} - -/** -* Wrapper to know if the build should enalbe the package stage -*/ -def isPackageEnabled() { - return env.PACKAGING_CHANGES == "true" || env.GITHUB_COMMENT?.contains('package') || matchesPrLabel(label: 'ci:package') -} - -/** -* Wrapper to know if the build should enable the windows extended support -*/ -def isExtendedWindowsEnabled() { - return env.EXT_WINDOWS_CHANGES == "true" || params.extended_windows_ci || env.GITHUB_COMMENT?.contains('extended windows') || matchesPrLabel(label: 'ci:extended-windows') -} - -/** -* Wrapper to know if the build should enable the M1 extended support -*/ -def isExtendedM1Enabled() { - return env.EXT_M1_CHANGES == "true" || params.extended_m1_ci || env.GITHUB_COMMENT?.contains('extended m1') || matchesPrLabel(label: 'ci:extended-m1') -} diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml deleted file mode 100644 index a2bcaae57e1..00000000000 --- a/.ci/jobs/defaults.yml +++ /dev/null @@ -1,19 +0,0 @@ - ---- - -##### GLOBAL METADATA - -- meta: - cluster: fleet-ci - -##### JOB DEFAULTS - -- job: - logrotate: - numToKeep: 20 - node: linux - concurrent: true - publishers: - - email: - recipients: infra-root+build@elastic.co - prune-dead-branches: true diff --git a/.ci/jobs/elastic-agent-mbp.yml b/.ci/jobs/elastic-agent-mbp.yml deleted file mode 100644 index 8947d15880a..00000000000 --- a/.ci/jobs/elastic-agent-mbp.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- job: - name: "elastic-agent/elastic-agent-mbp" - display-name: elastic-agent - description: "Elastic agent" - project-type: multibranch - script-path: .ci/Jenkinsfile - scm: - - github: - branch-discovery: no-pr - discover-pr-forks-strategy: merge-current - discover-pr-forks-trust: permission - discover-pr-origin: merge-current - discover-tags: true - head-filter-regex: '(main|7\.17|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' - notification-context: 'fleet-ci' - repo: elastic-agent - repo-owner: elastic - credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken - ssh-checkout: - credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba - build-strategies: - - tags: - ignore-tags-older-than: -1 - ignore-tags-newer-than: -1 - - regular-branches: true - - change-request: - ignore-target-only-changes: true - clean: - after: true - before: true - prune: true - shallow-clone: true - depth: 4 - do-not-fetch-tags: true - submodule: - disable: false - recursive: true - parent-credentials: true - timeout: 100 - timeout: '15' - use-author: true - wipe-workspace: true diff --git a/.ci/jobs/elastic-agent-schedule-daily.yml b/.ci/jobs/elastic-agent-schedule-daily.yml deleted file mode 100644 index 2fd728a9e8c..00000000000 --- a/.ci/jobs/elastic-agent-schedule-daily.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- job: - name: "elastic-agent/elastic-agent-schedule-daily" - display-name: Jobs scheduled daily (weekdays) - description: Jobs scheduled daily (weekdays) - view: Beats - project-type: pipeline - parameters: - - string: - name: branch_specifier - default: main - description: the Git branch specifier to build - pipeline-scm: - script-path: .ci/schedule-daily.groovy - scm: - - git: - url: git@github.com:elastic/elastic-agent.git - refspec: +refs/heads/*:refs/remotes/origin/* - wipe-workspace: 'True' - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - branches: - - $branch_specifier - triggers: - - timed: 'H H(2-3) * * 1-5' diff --git a/.ci/jobs/folders.yml b/.ci/jobs/folders.yml deleted file mode 100644 index cf309e259a3..00000000000 --- a/.ci/jobs/folders.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- job: - name: "elastic-agent" - description: "Elastic Agent" - project-type: folder diff --git a/.ci/schedule-daily.groovy b/.ci/schedule-daily.groovy deleted file mode 100644 index 70ad931ecda..00000000000 --- a/.ci/schedule-daily.groovy +++ /dev/null @@ -1,48 +0,0 @@ -@Library('apm@current') _ - -pipeline { - agent none - environment { - NOTIFY_TO = credentials('notify-to') - PIPELINE_LOG_LEVEL = 'INFO' - } - options { - timeout(time: 1, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - } - triggers { - cron('H H(2-3) * * 1-5') - } - stages { - stage('Nighly beats builds') { - steps { - runBuilds(quietPeriodFactor: 2000, branches: ['main', '8.', '8.']) - } - } - } - post { - cleanup { - notifyBuildResult(prComment: false) - } - } -} - -def runBuilds(Map args = [:]) { - def branches = getBranchesFromAliases(aliases: args.branches) - def quietPeriod = 0 - branches.each { branch -> - build(quietPeriod: quietPeriod, - job: "elastic-agent/elastic-agent-mbp/${branch}", - parameters: [ - booleanParam(name: 'integration_tests_ci', value: true), - // Disable running e2e until we fix the 2e2 testing suite - booleanParam(name: 'end_to_end_tests_ci', value: false) - ], - wait: false, propagate: false) - quietPeriod += args.quietPeriodFactor - } -} diff --git a/.ci/scripts/install-go.bat b/.ci/scripts/install-go.bat deleted file mode 100755 index 29448bd4f63..00000000000 --- a/.ci/scripts/install-go.bat +++ /dev/null @@ -1,57 +0,0 @@ -set GOPATH=%WORKSPACE% -set MAGEFILE_CACHE=%WORKSPACE%\.magefile - -set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH% - -curl --version >nul 2>&1 && ( - echo found curl -) || ( - choco install curl -y --no-progress --skipdownloadcache -) - -mkdir %WORKSPACE%\bin - -IF EXIST "%PROGRAMFILES(X86)%" ( - REM Force the gvm installation. - SET GVM_BIN=gvm.exe - curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe - IF ERRORLEVEL 1 ( - REM gvm installation has failed. - del bin\gvm.exe /s /f /q - exit /b 1 - ) -) ELSE ( - REM Windows 7 workers got a broken gvm installation. - curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe - IF ERRORLEVEL 1 ( - REM gvm installation has failed. - del bin\gvm.exe /s /f /q - exit /b 1 - ) -) - -SET GVM_BIN=gvm.exe -WHERE /q %GVM_BIN% -%GVM_BIN% version - -REM Install the given go version -%GVM_BIN% --debug install %GO_VERSION% - -REM Configure the given go version -FOR /f "tokens=*" %%i IN ('"%GVM_BIN%" use %GO_VERSION% --format=batch') DO %%i - -go env -IF ERRORLEVEL 1 ( - REM go is not configured correctly. - rmdir %WORKSPACE%\.gvm /s /q - exit /b 1 -) - -where mage -mage -version -IF ERRORLEVEL 1 ( - go get github.com/magefile/mage - IF ERRORLEVEL 1 ( - exit /b 1 - ) -) diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh deleted file mode 100755 index 31566c08726..00000000000 --- a/.ci/scripts/install-go.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -exuo pipefail - -MSG="environment variable missing" -GO_VERSION=${GO_VERSION:?$MSG} -PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"} -HOME=${HOME:?$MSG} -OS=$(uname -s| tr '[:upper:]' '[:lower:]') -ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') -GVM_CMD="${HOME}/bin/gvm" - -if command -v go -then - set +e - echo "Found Go. Checking version.." - FOUND_GO_VERSION=$(go version|awk '{print $3}'|sed s/go//) - if [ "$FOUND_GO_VERSION" == "$GO_VERSION" ] - then - echo "Versions match. No need to install Go. Exiting." - exit 0 - fi - set -e -fi - -if [ "${ARCH}" == "aarch64" ] ; then - GVM_ARCH_SUFFIX=arm64 -elif [ "${ARCH}" == "x86_64" ] ; then - GVM_ARCH_SUFFIX=amd64 -elif [ "${ARCH}" == "i686" ] ; then - GVM_ARCH_SUFFIX=386 -else - GVM_ARCH_SUFFIX=arm -fi - -echo "UNMET DEP: Installing Go" -mkdir -p "${HOME}/bin" - -curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-${OS}-${GVM_ARCH_SUFFIX}" -chmod +x "${GVM_CMD}" - -${GVM_CMD} "${GO_VERSION}" |cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE} - -eval "$("${GVM_CMD}" "${GO_VERSION}")" diff --git a/.ci/bump-golang.yml b/.github/updatecli-bump-golang.yml similarity index 98% rename from .ci/bump-golang.yml rename to .github/updatecli-bump-golang.yml index 470c6f4c8d5..df5d8104048 100644 --- a/.ci/bump-golang.yml +++ b/.github/updatecli-bump-golang.yml @@ -1,3 +1,4 @@ +# update-cli configuration for automated go updates --- name: Bump golang-version to latest version diff --git a/.github/workflows/bump-golang.yml b/.github/workflows/bump-golang.yml index 30b8fbd89b3..34d488b9d72 100644 --- a/.github/workflows/bump-golang.yml +++ b/.github/workflows/bump-golang.yml @@ -4,7 +4,7 @@ name: bump-golang on: workflow_dispatch: schedule: - - cron: '0 20 * * 6' + - cron: "0 20 * * 6" permissions: contents: read @@ -13,7 +13,6 @@ jobs: bump: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current @@ -21,4 +20,4 @@ jobs: vaultUrl: ${{ secrets.VAULT_ADDR }} vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - pipeline: ./.ci/bump-golang.yml + pipeline: ./.github/updatecli-bump-golang.yml