diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000000..5c5c239e4f59 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +properties([ + // max 5 build/branch/day + rateLimitBuilds(throttle: [count: 5, durationName: 'day', userBoost: true]), + // do not run multiple testruns on the same branch + disableConcurrentBuilds(), + parameters([ + string(name: 'SPLIT', defaultValue: '20', description: 'Number of buckets to split tests into.'), + string(name: 'OPTS', defaultValue: '', description: 'additional maven opts'), + ]) +]) + +this.prHead = null; +def checkPrHead() { + if(env.CHANGE_ID) { + println("checkPrHead - prHead:" + prHead) + println("checkPrHead - prHead2:" + pullRequest.head) + if (prHead == null) { + prHead = pullRequest.head; + } else { + if(prHead != pullRequest.head) { + currentBuild.result = 'ABORTED' + error('Found new changes on PR; aborting current build') + } + } + } +} +checkPrHead() + +def setPrLabel(String prLabel) { + if (env.CHANGE_ID) { + def mapping=[ + "SUCCESS":"tests passed", + "UNSTABLE":"tests unstable", + "FAILURE":"tests failed", + "PENDING":"tests pending", + ] + def newLabels = [] + for( String l : pullRequest.labels ) + newLabels.add(l) + for( String l : mapping.keySet() ) + newLabels.remove(mapping[l]) + newLabels.add(mapping[prLabel]) + echo ('' +newLabels) + pullRequest.labels=newLabels + } +} + +setPrLabel("PENDING"); + +def executorNode(run) { + hdbPodTemplate { + timeout(time: 24, unit: 'HOURS') { + node(POD_LABEL) { + container('hdb') { + run() + } + } + } + } +} + +def buildHive(args) { + configFileProvider([configFile(fileId: 'artifactory', variable: 'SETTINGS')]) { + withEnv(["MULTIPLIER=$params.MULTIPLIER","M_OPTS=$params.OPTS"]) { + sh '''#!/bin/bash -e +ls -l +set -x +. /etc/profile.d/confs.sh +export USER="`whoami`" +export MAVEN_OPTS="-Xmx2g" +export -n HIVE_CONF_DIR +cp $SETTINGS .git/settings.xml +OPTS=" -s $PWD/.git/settings.xml -B -Dtest.groups= " +OPTS+=" -Pitests,qsplits,dist" +OPTS+=" -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugin.surefire.SurefirePlugin=INFO" +OPTS+=" -Dmaven.repo.local=$PWD/.git/m2" +git config extra.mavenOpts "$OPTS" +OPTS=" $M_OPTS -Dmaven.test.failure.ignore " +if [ -s inclusions.txt ]; then OPTS+=" -Dsurefire.includesFile=$PWD/inclusions.txt";fi +if [ -s exclusions.txt ]; then OPTS+=" -Dsurefire.excludesFile=$PWD/exclusions.txt";fi +mvn $OPTS '''+args+''' +du -h --max-depth=1 +df -h +''' + } + } +} + +def hdbPodTemplate(closure) { + podTemplate( + containers: [ + containerTemplate(name: 'hdb', image: 'kgyrtkirk/hive-dev-box:executor', ttyEnabled: true, command: 'cat', + alwaysPullImage: true, + resourceRequestCpu: '1800m', + resourceLimitCpu: '8000m', + resourceRequestMemory: '6400Mi', + resourceLimitMemory: '12000Mi' + ), + ], yaml:''' +spec: + securityContext: + fsGroup: 1000 + tolerations: + - key: "type" + operator: "Equal" + value: "slave" + effect: "PreferNoSchedule" + - key: "type" + operator: "Equal" + value: "slave" + effect: "NoSchedule" + nodeSelector: + type: slave +''') { + closure(); + } +} + +def jobWrappers(closure) { + def finalLabel="FAILURE"; + try { + // allocate 1 precommit token for the execution + lock(label:'hive-precommit', quantity:1, variable: 'LOCKED_RESOURCE') { + timestamps { + echo env.LOCKED_RESOURCE + checkPrHead() + closure() + } + } + finalLabel=currentBuild.currentResult + } finally { + setPrLabel(finalLabel) + } +} + +def saveWS() { + sh '''#!/bin/bash -e + tar --exclude=archive.tar -cf archive.tar . + ls -l archive.tar + rsync -rltDq --stats archive.tar rsync://rsync/data/$LOCKED_RESOURCE''' +} + +def loadWS() { + sh '''#!/bin/bash -e + rsync -rltDq --stats rsync://rsync/data/$LOCKED_RESOURCE archive.tar + tar -xf archive.tar''' +} + +jobWrappers { + + def splits + executorNode { + container('hdb') { + stage('Checkout') { + checkout scm + } + stage('Compile') { + buildHive("install -Dtest=noMatches") + } + checkPrHead() + stage('Upload') { + saveWS() + sh '''#!/bin/bash -e + # make parallel-test-execution plugins source scanner happy ~ better results for 1st run + find . -name '*.java'|grep /Test|grep -v src/test/java|grep org/apache|while read f;do t="`echo $f|sed 's|.*org/apache|happy/src/test/java/org/apache|'`";mkdir -p "${t%/*}";touch "$t";done + ''' + splits = splitTests parallelism: count(Integer.parseInt(params.SPLIT)), generateInclusions: true, estimateTestsFromFiles: true + } + } + } + + stage('Testing') { + + def branches = [:] + for (int i = 0; i < splits.size(); i++) { + def num = i + def split = splits[num] + def splitName=String.format("split-%02d",num+1) + branches[splitName] = { + executorNode { + stage('Prepare') { + loadWS(); + writeFile file: (split.includes ? "inclusions.txt" : "exclusions.txt"), text: split.list.join("\n") + writeFile file: (split.includes ? "exclusions.txt" : "inclusions.txt"), text: '' + sh '''echo "@INC";cat inclusions.txt;echo "@EXC";cat exclusions.txt;echo "@END"''' + } + try { + stage('Test') { + buildHive("org.apache.maven.plugins:maven-antrun-plugin:run@{define-classpath,setup-test-dirs,setup-metastore-scripts} org.apache.maven.plugins:maven-surefire-plugin:test -q") + } + } finally { + stage('Archive') { + junit '**/TEST-*.xml' + } + } + } + } + } + parallel branches + } +}