From 4851ff5231bad6fcb9f57eba047e292ace95881f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 17 Feb 2021 12:38:55 +0000 Subject: [PATCH] [JCasC] google cloud engine local setup (#979) * [JCasC] google cloud engine local setup * Changed cloud compute entry * change images * ci(jenkins): test scenarios for analysis the GCE * Test pipelines: in batches of 10 workers, in batches of 10*i workers, in batches of 100*i workers * Enable less delay when provisioining and a longer timeout * Use latest available jenkins docker image Use a different template with more disk Use vault credentials * Add Docs Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> (cherry picked from commit a4764108312646daa2bb585bfd8bcf7564cd098e) --- docs/CLOUD.md | 33 +++++++++ local/Dockerfile | 2 +- local/configs/google.yaml | 45 ++++++++++++ local/configs/jenkins.yaml | 2 + local/configs/plugins.txt | 4 ++ local/docker-compose.yml | 10 ++- .../resources/jobs/gce-test-orchestrator.dsl | 49 +++++++++++++ src/test/resources/jobs/gce-test.dsl | 68 +++++++++++++++++++ 8 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 docs/CLOUD.md create mode 100644 local/configs/google.yaml create mode 100644 src/test/resources/jobs/gce-test-orchestrator.dsl create mode 100644 src/test/resources/jobs/gce-test.dsl diff --git a/docs/CLOUD.md b/docs/CLOUD.md new file mode 100644 index 000000000..bb497436a --- /dev/null +++ b/docs/CLOUD.md @@ -0,0 +1,33 @@ +# Cloud + +We are using the Google Compute Jenkins plugins. + +See https://github.com/jenkinsci/google-compute-engine-plugin/#configuration-as-code-support for further examples. + +## How to + +### Add a new AMI + +You can add new VMs, for such you need to: + +1. Create a new template in the [Google Cloud](https://console.cloud.google.com/compute/instanceTemplates/list?project=elastic-observability) +1. Add a new entry in `local/configs/google.yaml`, that's the JCasC for the cloud provider. +1. If you prefer you can use the Jenkins UI. Go to the [configureClouds](http://localhost:18080/configureClouds/) menu. + +Once you are happy with the configuration then you can export to a JCasC format, for such you just need to go to +[JCasC](http://localhost:18080/configuration-as-code/) and click on the `View Configuration` button. + + +### Modify the existing credentials for the google service account + +If for any reason you'd like to update the existing credentials then you need to follow the below steps: + +1. Go to [Service Accounts](https://console.cloud.google.com/iam-admin/serviceaccounts?project=elastic-observability) +1. Select the `jenkins-gce` one. +1. Create a new JSON key type. +1. Transform to base64 and update the existing vault entry + +```bash +$ base64 -i elastic-observability-*********.json -o base64.json +$ vault write secret/observability-team/ci/service-account/jenkins-gce-elastic-observability google_cloud_secret=@base64.json ticket=https://github.com/elastic/apm-pipeline-library/pull/356 +``` diff --git a/local/Dockerfile b/local/Dockerfile index 517cfe2da..b42319ffd 100644 --- a/local/Dockerfile +++ b/local/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/infra/jenkins:202010021728.bc28cd532cad +FROM docker.elastic.co/infra/jenkins:202102102240.c37e3f69e293 COPY configs/plugins.txt /usr/share/jenkins/ref/plugins.txt RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt diff --git a/local/configs/google.yaml b/local/configs/google.yaml new file mode 100644 index 000000000..031d213e3 --- /dev/null +++ b/local/configs/google.yaml @@ -0,0 +1,45 @@ +credentials: + system: + domainCredentials: + - credentials: + ## See https://github.com/jenkinsci/google-oauth-plugin/blob/google-oauth-plugin-1.0.0/src/test/resources/com/google/jenkins/plugins/credentials/oauth/json-service-account-config.yml + - googleRobotPrivateKey: + projectId: 'elastic-observability' + serviceAccountConfig: + json: + secretJsonKey: ${google_cloud_secret} + +jenkins: + clouds: + - computeEngine: + cloudName: "elastic-observability" + configurations: + - bootDiskAutoDelete: true + bootDiskSizeGb: 50 + bootDiskSourceImageProject: "elastic-observability" + bootDiskType: "https://www.googleapis.com/compute/v1/projects/elastic-observability/zones/us-central1-f/diskTypes/pd-ssd" + description: "obs11 workers" + externalAddress: true + javaExecPath: "java" + labels: "linux immutable obs11" + launchTimeoutSeconds: 600 + machineType: "https://www.googleapis.com/compute/v1/projects/elastic-observability/zones/us-central1-f/machineTypes/g1-small" + mode: NORMAL + namePrefix: "obs11-linux" + networkConfiguration: + autofilled: + network: "https://www.googleapis.com/compute/v1/projects/elastic-observability/global/networks/default" + subnetwork: "https://www.googleapis.com/compute/v1/projects/elastic-observability/regions/us-central1/subnetworks/default" + numExecutors: 1 + oneShot: true + preemptible: true + region: "https://www.googleapis.com/compute/v1/projects/elastic-observability/regions/us-central1" + retentionTimeMinutes: 10 + runAsUser: "jenkins-gce" + serviceAccountEmail: "jenkins-gce@elastic-observability.iam.gserviceaccount.com" + template: "https://www.googleapis.com/compute/v1/projects/elastic-observability/global/instanceTemplates/elastic-apm-ci-ubuntu-1804-lts-20210201222928" + zone: "https://www.googleapis.com/compute/v1/projects/elastic-observability/zones/us-central1-f" + credentialsId: "elastic-observability" + instanceCapStr: "300" + instanceId: "20d4387f-26d5-470f-8455-d8e4c15de6df" + projectId: "elastic-observability" diff --git a/local/configs/jenkins.yaml b/local/configs/jenkins.yaml index 1d26ac24c..0458fbc90 100644 --- a/local/configs/jenkins.yaml +++ b/local/configs/jenkins.yaml @@ -124,6 +124,8 @@ jobs: - file: "/var/pipeline-library/src/test/resources/jobs/dockerLogin.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/dockerLogs.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/downstream.dsl" + - file: "/var/pipeline-library/src/test/resources/jobs/gce-test.dsl" + - file: "/var/pipeline-library/src/test/resources/jobs/gce-test-orchestrator.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/abort.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/connectionRefused.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/error.dsl" diff --git a/local/configs/plugins.txt b/local/configs/plugins.txt index b6d9d6815..972e3d7c5 100644 --- a/local/configs/plugins.txt +++ b/local/configs/plugins.txt @@ -1,5 +1,9 @@ disable-github-multibranch-status filesystem_scm +google-compute-engine job-dsl +metrics +monitoring opentelemetry::https://storage.googleapis.com/apm-ci-temp/plugins/opentelemetry.hpi +plot ssh-agent diff --git a/local/docker-compose.yml b/local/docker-compose.yml index 17800db3f..a2e3329fb 100644 --- a/local/docker-compose.yml +++ b/local/docker-compose.yml @@ -8,12 +8,17 @@ services: CASC_JENKINS_CONFIG: /var/jenkins_home/casc_configs CASC_VAULT_ENGINE_VERSION: "1" CASC_VAULT_PATHS: | - secret/jcasc/localhost/base,secret/jcasc/localhost/apm-ci,secret/observability-team/ci/service-account/jenkins-google-storage-elastic-observability + secret/jcasc/localhost/base,secret/jcasc/localhost/apm-ci,secret/observability-team/ci/service-account/jenkins-google-storage-elastic-observability,secret/observability-team/ci/service-account/jenkins-gce-elastic-observability CASC_VAULT_TOKEN: ${VAULT_TOKEN} CASC_VAULT_URL: ${VAULT_ADDR:-https://secrets.elastic.co:8200} JAVA_OPTS: >- -Djenkins.install.runSetupWizard=false -Dhudson.model.UsageStatistics.disabled=true + -Dhudson.slaves.NodeProvisioner.initialDelay=0 + -Dhudson.slaves.NodeProvisioner.MARGIN=100 + -Dhudson.slaves.NodeProvisioner.MARGIN0=1.0 + -Dhudson.model.LoadStatistics.decay=0.5 + -Dhudson.model.LoadStatistics.clock=5000 volumes: - type: volume source: jenkins_home @@ -21,6 +26,9 @@ services: - type: bind source: ./configs/jenkins.yaml target: /var/jenkins_home/casc_configs/jenkins.yaml + - type: bind + source: ./configs/google.yaml + target: /var/jenkins_home/casc_configs/google.yaml read_only: true - type: bind source: ../. diff --git a/src/test/resources/jobs/gce-test-orchestrator.dsl b/src/test/resources/jobs/gce-test-orchestrator.dsl new file mode 100644 index 000000000..7d1a49775 --- /dev/null +++ b/src/test/resources/jobs/gce-test-orchestrator.dsl @@ -0,0 +1,49 @@ +NAME = 'it/gce-test-orchestrator' +DSL = ''' +pipeline { + agent none + stages { + /*stage('Test for a while') { + steps { + parallelStepsForAWhile() + } + }*/ + stage('Test in batches') { + steps { + buildCall(10, 10) + } + } + stage('Test in massive requests') { + steps { + buildCall(3, 100) + } + } + } +} + +def buildCall(num, factor) { + for (i = 1; i < num; i++) { + buildRun(i * factor, false) + } +} + +def buildRun(i, wait) { + build(job: 'gce-test', propagate: false, quietPeriod: 0, wait: wait, + parameters: [string(name: 'num', value: "${i}")]) +} + +// It runs in batches of 10 node requests +def parallelStepsForAWhile(){ + for (i = 0; i < 100; i++) { + buildRun(10, false) + } +} +''' + +pipelineJob(NAME) { + definition { + cps { + script(DSL.stripIndent()) + } + } +} diff --git a/src/test/resources/jobs/gce-test.dsl b/src/test/resources/jobs/gce-test.dsl new file mode 100644 index 000000000..9e98ae02d --- /dev/null +++ b/src/test/resources/jobs/gce-test.dsl @@ -0,0 +1,68 @@ +NAME = 'it/gce-test' +DSL = ''' +import groovy.time.* +pipeline { + agent none + environment { + PIPELINE_LOG_LEVEL = 'INFO' + HOME = "${WORKSPACE}" + } + parameters { + string defaultValue: '1', description: '', name: 'num', trim: false + } + options { + timeout(time: 2, unit: 'HOURS') + timestamps() + disableResume() + durabilityHint('PERFORMANCE_OPTIMIZED') + disableConcurrentBuilds() + } + stages { + stage('Test') { + steps { + script { + currentBuild.description = "With ${params.num} nodes" + } + parallelSteps("${params.num}".toString().toInteger()) + } + } + } +} + +def parallelSteps(max){ + def num = (max < 1) ? 1 : max + def parallelSteps = [:] + for (i = 0; i < num; i++) { + parallelSteps["step-${i}"] = { + echo 'parallelSteps|started' + TimeDuration timeDuration = elapsedTime { + node('linux && immutable'){ + echo 'hey!' + } + } + echo "parallelSteps|finished|${timeDuration.toString()}" + } + } + parallel parallelSteps +} + +def elapsedTime(Closure closure){ + def timeStart = new Date() + closure() + def timeStop = new Date() + TimeCategory.minus(timeStop, timeStart) +} + +''' + +pipelineJob(NAME) { + parameters { + stringParam('num', '1') + } + concurrentBuild(false) + definition { + cps { + script(DSL.stripIndent()) + } + } +}