From 82958ce0734e14355cc33b5d47fb43cc26658125 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 5 Mar 2025 21:22:39 +0000 Subject: [PATCH 1/5] Update OS core systemd related test commands Signed-off-by: Peter Zhu --- scripts/components/OpenSearch/integtest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/components/OpenSearch/integtest.sh b/scripts/components/OpenSearch/integtest.sh index 7f3a4b08c7..dce75c8696 100644 --- a/scripts/components/OpenSearch/integtest.sh +++ b/scripts/components/OpenSearch/integtest.sh @@ -13,7 +13,7 @@ echo "Check if distribution is deb or rpm on linux" if [ "$OSTYPE" = "linux-gnu" ]; then if (dpkg -s opensearch > /dev/null 2>&1) || (rpm -q opensearch > /dev/null 2>&1); then echo "Run systemd integTest for OpenSearch core engine" - ./gradlew :qa:systemd-test:test --tests org.opensearch.systemdinteg.SystemdIntegTests --console=plain + ./gradlew qa:systemd-test:integTest --tests org.opensearch.systemdinteg.SystemdIntegTests --console=plain else echo "No deb or rpm installed detected, skip test" fi From 90308c4413bd975a80bbaf13151e0d098f417884 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 5 Mar 2025 21:50:21 +0000 Subject: [PATCH 2/5] Update ubuntu2404 nodes Signed-off-by: Peter Zhu --- jenkins/docker/docker-build.jenkinsfile | 2 +- jenkins/docker/docker-copy.jenkinsfile | 2 +- jenkins/docker/docker-re-release.jenkinsfile | 2 +- jenkins/gradle/gradle-check.jenkinsfile | 8 ++++---- jenkins/packer/packer-build.jenkinsfile | 2 +- tests/jenkins/TestDockerBuild.groovy | 2 +- .../docker/docker-build.jenkinsfile.txt | 2 +- .../packer/packer-build.jenkinsfile.txt | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/jenkins/docker/docker-build.jenkinsfile b/jenkins/docker/docker-build.jenkinsfile index 0222bd3d00..4c54a587e0 100644 --- a/jenkins/docker/docker-build.jenkinsfile +++ b/jenkins/docker/docker-build.jenkinsfile @@ -18,7 +18,7 @@ def docker_args = [ ] def agent_nodes = [ - "linux": "Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder", + "linux": "Jenkins-Agent-Ubuntu2404-X64-M52xlarge-Docker-Builder", "windows": "Jenkins-Agent-Windows2019-X64-M54xlarge-Docker-Builder", ] diff --git a/jenkins/docker/docker-copy.jenkinsfile b/jenkins/docker/docker-copy.jenkinsfile index e3accf56a0..b4b28802fb 100644 --- a/jenkins/docker/docker-copy.jenkinsfile +++ b/jenkins/docker/docker-copy.jenkinsfile @@ -48,7 +48,7 @@ pipeline { stage("Image Copy") { agent { docker { - label 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder' + label 'Jenkins-Agent-Ubuntu2404-X64-M52xlarge-Docker-Builder' image 'opensearchstaging/ci-runner:ubuntu2404-x64-docker-buildx0.9.1-qemu8.2-v1' args '-u root -v /var/run/docker.sock:/var/run/docker.sock' registryUrl 'https://public.ecr.aws/' diff --git a/jenkins/docker/docker-re-release.jenkinsfile b/jenkins/docker/docker-re-release.jenkinsfile index cc027040b7..c87e455ef4 100644 --- a/jenkins/docker/docker-re-release.jenkinsfile +++ b/jenkins/docker/docker-re-release.jenkinsfile @@ -41,7 +41,7 @@ pipeline { stage("Docker re-release") { agent { docker { - label 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder' + label 'Jenkins-Agent-Ubuntu2404-X64-M52xlarge-Docker-Builder' image 'opensearchstaging/ci-runner:ubuntu2404-x64-docker-buildx0.9.1-qemu8.2-v1' args '-u root -v /var/run/docker.sock:/var/run/docker.sock' registryUrl 'https://public.ecr.aws/' diff --git a/jenkins/gradle/gradle-check.jenkinsfile b/jenkins/gradle/gradle-check.jenkinsfile index d266ed7572..7808c93fac 100644 --- a/jenkins/gradle/gradle-check.jenkinsfile +++ b/jenkins/gradle/gradle-check.jenkinsfile @@ -50,7 +50,7 @@ pipeline { choice( name: 'AGENT_LABEL', description: 'Choose which jenkins agent to run gradle check on', - choices: ['Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host', 'Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host'], + choices: ['Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host', 'Jenkins-Agent-Ubuntu2404-X64-M58xlarge-Single-Host', 'Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host'], ) } triggers { @@ -70,9 +70,9 @@ pipeline { printPostContent: false ) parameterizedCron ''' - H 3 * * * %GIT_REFERENCE=main;AGENT_LABEL=Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host - H 6 * * * %GIT_REFERENCE=1.x;AGENT_LABEL=Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host - H 9 * * * %GIT_REFERENCE=2.x;AGENT_LABEL=Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host + H 3 * * * %GIT_REFERENCE=main;AGENT_LABEL=Jenkins-Agent-Ubuntu2404-X64-M58xlarge-Single-Host + H 6 * * * %GIT_REFERENCE=2.19;AGENT_LABEL=Jenkins-Agent-Ubuntu2404-X64-M58xlarge-Single-Host + H 9 * * * %GIT_REFERENCE=2.x;AGENT_LABEL=Jenkins-Agent-Ubuntu2404-X64-M58xlarge-Single-Host ''' } environment { diff --git a/jenkins/packer/packer-build.jenkinsfile b/jenkins/packer/packer-build.jenkinsfile index 66d0adc20a..d8ebf27a5e 100644 --- a/jenkins/packer/packer-build.jenkinsfile +++ b/jenkins/packer/packer-build.jenkinsfile @@ -18,7 +18,7 @@ pipeline { } agent { docker { - label 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder' + label 'Jenkins-Agent-Ubuntu2404-X64-M52xlarge-Docker-Builder' image 'opensearchstaging/ci-runner:ubuntu2404-x64-docker-buildx0.9.1-qemu8.2-v1' registryUrl 'https://public.ecr.aws/' alwaysPull true diff --git a/tests/jenkins/TestDockerBuild.groovy b/tests/jenkins/TestDockerBuild.groovy index a89c789a3c..010b5330db 100644 --- a/tests/jenkins/TestDockerBuild.groovy +++ b/tests/jenkins/TestDockerBuild.groovy @@ -68,7 +68,7 @@ class TestDockerBuild extends BuildPipelineTest { assertCallStack().contains("docker-build.sh(echo Account: jenkins-staging-dockerhub-credential)") // Make sure dockerBuildOS is deciding agent_node docker_nodes docker_args correctly - assertCallStack().contains("docker-build.echo(Executing on agent [docker:[alwaysPull:true, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2404-x64-docker-buildx0.9.1-qemu8.2-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])") + assertCallStack().contains("docker-build.echo(Executing on agent [docker:[alwaysPull:true, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2404-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2404-x64-docker-buildx0.9.1-qemu8.2-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])") printCallStack() } diff --git a/tests/jenkins/jenkinsjob-regression-files/docker/docker-build.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/docker/docker-build.jenkinsfile.txt index 88988168c0..aadfeb89c2 100644 --- a/tests/jenkins/jenkinsjob-regression-files/docker/docker-build.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/docker/docker-build.jenkinsfile.txt @@ -5,7 +5,7 @@ docker-build.stage(Parameters Check, groovy.lang.Closure) docker-build.script(groovy.lang.Closure) docker-build.stage(docker-build, groovy.lang.Closure) - docker-build.echo(Executing on agent [docker:[alwaysPull:true, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2404-x64-docker-buildx0.9.1-qemu8.2-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) + docker-build.echo(Executing on agent [docker:[alwaysPull:true, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2404-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2404-x64-docker-buildx0.9.1-qemu8.2-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) docker-build.script(groovy.lang.Closure) docker-build.echo(The docker-build workflow will only push docker images to staging, please use docker-copy to move the image to other repositories) docker-build.checkout({$class=GitSCM, branches=[{name=main}], userRemoteConfigs=[{url=https://github.com/opensearch-project/opensearch-build}]}) diff --git a/tests/jenkins/jenkinsjob-regression-files/packer/packer-build.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/packer/packer-build.jenkinsfile.txt index 2faa9483bf..aff0e1a58e 100644 --- a/tests/jenkins/jenkinsjob-regression-files/packer/packer-build.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/packer/packer-build.jenkinsfile.txt @@ -3,7 +3,7 @@ packer-build.library({identifier=jenkins@2.2.0, retriever=null}) packer-build.pipeline(groovy.lang.Closure) packer-build.timeout({time=4, unit=HOURS}) - packer-build.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2404-x64-docker-buildx0.9.1-qemu8.2-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) + packer-build.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2404-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2404-x64-docker-buildx0.9.1-qemu8.2-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) packer-build.stage(Parameters Check, groovy.lang.Closure) packer-build.script(groovy.lang.Closure) packer-build.stage(packer-build, groovy.lang.Closure) From 59a2c40b86ee810cdab9d01efb7bc2a9131e43b2 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 6 Mar 2025 00:20:09 +0000 Subject: [PATCH 3/5] Update trivy clean up commands here Signed-off-by: Peter Zhu --- jenkins/docker/docker-scan.jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/docker/docker-scan.jenkinsfile b/jenkins/docker/docker-scan.jenkinsfile index 4bdaf475be..bd8d421c10 100644 --- a/jenkins/docker/docker-scan.jenkinsfile +++ b/jenkins/docker/docker-scan.jenkinsfile @@ -67,7 +67,7 @@ pipeline { archiveArtifacts artifacts: 'scan_docker_image.*' script { postCleanup() - sh("trivy image --clear-cache") + sh("trivy clean --all") } } } From e33eedbd55a020e390bd9d65cdbb588ebc753c58 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 6 Mar 2025 00:44:18 +0000 Subject: [PATCH 4/5] Update tests Signed-off-by: Peter Zhu --- .../docker/docker-scan.jenkinsfile.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jenkins/jenkinsjob-regression-files/docker/docker-scan.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/docker/docker-scan.jenkinsfile.txt index 5226aacb4b..6ed6bfe97d 100644 --- a/tests/jenkins/jenkinsjob-regression-files/docker/docker-scan.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/docker/docker-scan.jenkinsfile.txt @@ -21,4 +21,4 @@ docker-scan.script(groovy.lang.Closure) docker-scan.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) - docker-scan.sh(trivy image --clear-cache) + docker-scan.sh(trivy clean --all) From d2ef1af4c60338fab105815c2457fafcbd2736c7 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 6 Mar 2025 13:38:52 -0500 Subject: [PATCH 5/5] Update docker scan with new lib 8.2.2 Signed-off-by: Peter Zhu --- jenkins/docker/docker-scan.jenkinsfile | 2 +- manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1.yml | 2 +- manifests/3.0.0-beta1/opensearch-3.0.0-beta1.yml | 2 +- tests/jenkins/TestDockerScanJob.groovy | 2 +- .../docker/docker-scan.jenkinsfile.txt | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jenkins/docker/docker-scan.jenkinsfile b/jenkins/docker/docker-scan.jenkinsfile index bd8d421c10..46eb33b5c6 100644 --- a/jenkins/docker/docker-scan.jenkinsfile +++ b/jenkins/docker/docker-scan.jenkinsfile @@ -7,7 +7,7 @@ * compatible open source license. */ -lib = library(identifier: 'jenkins@1.0.4', retriever: modernSCM([ +lib = library(identifier: 'jenkins@8.2.2', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ])) diff --git a/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1.yml b/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1.yml index 174f17adbc..c4c48d2a86 100644 --- a/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1.yml +++ b/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1.yml @@ -7,7 +7,7 @@ build: ci: image: name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 - args: -e JAVA_HOME=/opt/java/openjdk-23 + args: -e JAVA_HOME=/opt/java/openjdk-21 components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git diff --git a/manifests/3.0.0-beta1/opensearch-3.0.0-beta1.yml b/manifests/3.0.0-beta1/opensearch-3.0.0-beta1.yml index 1f10c9b79a..172231f565 100644 --- a/manifests/3.0.0-beta1/opensearch-3.0.0-beta1.yml +++ b/manifests/3.0.0-beta1/opensearch-3.0.0-beta1.yml @@ -7,7 +7,7 @@ build: ci: image: name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 - args: -e JAVA_HOME=/opt/java/openjdk-23 + args: -e JAVA_HOME=/opt/java/openjdk-21 components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git diff --git a/tests/jenkins/TestDockerScanJob.groovy b/tests/jenkins/TestDockerScanJob.groovy index ee23e098cb..0e6f12fc25 100644 --- a/tests/jenkins/TestDockerScanJob.groovy +++ b/tests/jenkins/TestDockerScanJob.groovy @@ -20,7 +20,7 @@ class TestDockerScanJob extends BuildPipelineTest { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('1.0.4') + .defaultVersion('8.2.2') .allowOverride(true) .implicit(true) .targetPath('vars') diff --git a/tests/jenkins/jenkinsjob-regression-files/docker/docker-scan.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/docker/docker-scan.jenkinsfile.txt index 6ed6bfe97d..a7987c9ffd 100644 --- a/tests/jenkins/jenkinsjob-regression-files/docker/docker-scan.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/docker/docker-scan.jenkinsfile.txt @@ -1,6 +1,6 @@ docker-scan.run() docker-scan.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - docker-scan.library({identifier=jenkins@1.0.4, retriever=null}) + docker-scan.library({identifier=jenkins@8.2.2, retriever=null}) docker-scan.pipeline(groovy.lang.Closure) docker-scan.timeout({time=30}) docker-scan.echo(Executing on agent [label:none]) @@ -11,7 +11,7 @@ docker-scan.scanDockerImage({imageFullName=alpine:3, imageResultFile=scan_docker_image}) scanDockerImage.sh( touch scan_docker_image.txt scan_docker_image.json - trivy image --clear-cache + trivy clean --all docker rmi `docker images -f "dangling=true" -q` || echo docker rmi alpine:3 || echo trivy image --format table --output scan_docker_image.txt alpine:3