Publish artifacts from release branches separately (#8062) (#8159)

By default, the build-ova.sh script used by vic-product to create the
unified OVA picks the most recent build uploaded to vic-engine-builds.

Because builds from all branches are written to the same directory,
this can lead to unexpected combinations of components being assembled
into an OVA.

To allow downstream jobs more reliably consume desired dependencies,
publish artifacts from release branches into sub-directories of a
releases directory.

This alone will improve the behavior of downstream builds expecting to
consume dependencies from master. With additional changes, this can be
leveraged to allow push builds on release branches to behave more like
push builds on master: consuming the most up-to-date build from the
appropriate branch of the upstream.

Update jenkins-nightly-run.sh to allow download of builds published to
release branch directories.

Additionally, publish artifacts from PR builds into separate bucket.

(cherry picked from commit 00d377d)

.drone.yml

@@ -142,7 +142,7 @@ pipeline:
       - 'ls -la ./$BIN && ./$BIN/ui/sync-vic-ui-version.sh -p bin/ 2>&1'
       - 'rm $BIN/vic_ui_*.tar.gz'
     when:
-      event: push
+      event: [push, pull_request]
       status: [success, failure]
 
   vic-ui-release:
@@ -204,24 +204,53 @@ pipeline:
       - 'cp $BIN/vic_${DRONE_BUILD_NUMBER}.tar.gz bundle-release/vic_$(git describe --tags --abbrev=0).tar.gz'
     when:
       repo: vmware/vic
-      branch: [master, 'releases/*']
-      event: [push, tag]
       status: [success, failure]
 
-  publish-gcs-builds-on-pass:
+  publish-gcs-builds-for-pr:
+    image: 'victest/drone-gcs:1'
+    pull: true
+    secrets:
+      - google_key
+    source: bundle
+    target: vic-engine-pr-builds/pull/${DRONE_PULL_REQUEST}/
+    acl:
+      - 'allUsers:READER'
+    cache_control: 'public,max-age=3600'
+    when:
+      repo: vmware/vic
+      event: pull_request
+      status: success
+
+  publish-gcs-builds-on-master-push-pass:
     image: 'victest/drone-gcs:1'
     pull: true
     secrets:
       - google_key
     source: bundle
-    target: vic-engine-builds
+    target: vic-engine-builds/
     acl:
       - 'allUsers:READER'
     cache_control: 'public,max-age=3600'
     when:
       repo: vmware/vic
       event: push
-      branch: [master, 'releases/*']
+      branch: [master]
+      status: success
+
+  publish-gcs-builds-on-release-push-pass:
+    image: 'victest/drone-gcs:1'
+    pull: true
+    secrets:
+      - google_key
+    source: bundle
+    target: vic-engine-builds/${DRONE_BRANCH}/
+    acl:
+      - 'allUsers:READER'
+    cache_control: 'public,max-age=3600'
+    when:
+      repo: vmware/vic
+      event: push
+      branch: ['releases/*']
       status: success
 
   publish-gcs-builds-on-fail:
@@ -230,7 +259,7 @@ pipeline:
     secrets:
       - google_key
     source: bundle
-    target: vic-engine-failed-builds
+    target: vic-engine-failed-builds/
     acl:
       - 'allUsers:READER'
     cache_control: 'public,max-age=3600'
@@ -246,7 +275,7 @@ pipeline:
     secrets:
       - google_key
     source: bundle-release
-    target: vic-engine-releases
+    target: vic-engine-releases/
     acl:
       - 'allUsers:READER'
     cache_control: 'public,max-age=3600'

tests/nightly/jenkins-nightly-run.sh

@@ -30,6 +30,7 @@ ESX_67_VERSION="ob-8169922"
 VC_67_VERSION="ob-8217866"
 
 DEFAULT_LOG_UPLOAD_DEST="vic-ci-logs"
+DEFAULT_VCH_BRANCH=""
 DEFAULT_VCH_BUILD="*"
 DEFAULT_TESTCASES=("tests/manual-test-cases/Group5-Functional-Tests" "tests/manual-test-cases/Group13-vMotion" "tests/manual-test-cases/Group21-Registries" "tests/manual-test-cases/Group23-Future-Tests")
 
@@ -55,7 +56,8 @@ testcases=("${@:-${DEFAULT_TESTCASES[@]}}")
 # TODO: the version downloaded by this logic is not coupled with the tests that will be run against it. This should be altered to pull a version that matches the commit SHA of the tests
 # we will be running or similar mechanism.
 VCH_BUILD=${VCH_BUILD:-${DEFAULT_VCH_BUILD}}
-input=$(gsutil ls -l gs://vic-engine-builds/${VIC_BINARY_PREFIX}${VCH_BUILD} | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | cut -d '/' -f 4)
+VCH_BRANCH=${VCH_BRANCH:-${DEFAULT_VCH_BRANCH}}
+input=$(gsutil ls -l gs://vic-engine-builds/${VCH_BRANCH}${VCH_BRANCH:+/}${VIC_BINARY_PREFIX}${VCH_BUILD} | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | cut -d '/' -f 4)
 
 # strip prefix and suffix from archive filename
 VCH_BUILD=${input#${VIC_BINARY_PREFIX}}