Merge pull request kubernetes#28782 from zmerlynn/automated-cherry-pi…

…ck-of-#28172-kubernetes#28193-upstream-release-1.2

Automated cherry pick of kubernetes#28172 kubernetes#28193

build/common.sh

@@ -1211,6 +1211,16 @@ function kube::release::gcs::copy_release_artifacts() {
   fi
 
   gsutil ls -lhr "${gcs_destination}" || return 1
+
+  if [[ -n "${KUBE_GCS_RELEASE_BUCKET_MIRROR:-}" ]]; then
+    local -r gcs_mirror="gs://${KUBE_GCS_RELEASE_BUCKET_MIRROR}/${KUBE_GCS_RELEASE_PREFIX}"
+    kube::log::status "Mirroring build to ${gcs_mirror}"
+    gsutil -q -m "${gcs_options[@]+${gcs_options[@]}}" rsync -d -r "${gcs_destination}" "${gcs_mirror}" || return 1
+    if [[ ${KUBE_GCS_MAKE_PUBLIC} =~ ^[yY]$ ]]; then
+      kube::log::status "Marking all uploaded mirror objects public"
+      gsutil -q -m acl ch -R -g all:R "${gcs_mirror}" >/dev/null 2>&1 || return 1
+    fi
+  fi
 }
 
 # Publish a new ci version, (latest,) but only if the release files actually
@@ -1475,7 +1485,19 @@ function kube::release::gcs::verify_ci_ge() {
 #   If new version is greater than the GCS version
 function kube::release::gcs::publish() {
   local -r publish_file="${1-}"
-  local -r publish_file_dst="gs://${KUBE_GCS_RELEASE_BUCKET}/${publish_file}"
+
+  kube::release::gcs::publish_to_bucket "${KUBE_GCS_RELEASE_BUCKET}" "${publish_file}" || return 1
+
+  if [[ -n "${KUBE_GCS_RELEASE_BUCKET_MIRROR:-}" ]]; then
+    kube::release::gcs::publish_to_bucket "${KUBE_GCS_RELEASE_BUCKET_MIRROR}" "${publish_file}" || return 1
+  fi
+}
+
+
+function kube::release::gcs::publish_to_bucket() {
+  local -r publish_bucket="${1}"
+  local -r publish_file="${2}"
+  local -r publish_file_dst="gs://${publish_bucket}/${publish_file}"
 
   mkdir -p "${RELEASE_STAGE}/upload" || return 1
   echo "${KUBE_GCS_PUBLISH_VERSION}" > "${RELEASE_STAGE}/upload/latest" || return 1
@@ -1488,7 +1510,7 @@ function kube::release::gcs::publish() {
     gsutil acl ch -R -g all:R "${publish_file_dst}" >/dev/null 2>&1 || return 1
     gsutil setmeta -h "Cache-Control:private, max-age=0" "${publish_file_dst}" >/dev/null 2>&1 || return 1
     # If public, validate public link
-    local -r public_link="https://storage.googleapis.com/${KUBE_GCS_RELEASE_BUCKET}/${publish_file}"
+    local -r public_link="https://storage.googleapis.com/${publish_bucket}/${publish_file}"
     kube::log::status "Validating uploaded version file at ${public_link}"
     contents="$(curl -s "${public_link}")"
   else

build/push-ci-build.sh

@@ -41,6 +41,7 @@ KUBE_GCS_MAKE_PUBLIC='y'
 KUBE_GCS_UPLOAD_RELEASE='y'
 KUBE_GCS_DELETE_EXISTING='y'
 KUBE_GCS_RELEASE_BUCKET='kubernetes-release'
+KUBE_GCS_RELEASE_BUCKET_MIRROR='kubernetes-release-dev'
 KUBE_GCS_RELEASE_PREFIX="ci/${LATEST}"
 KUBE_GCS_PUBLISH_VERSION="${LATEST}"
 

cluster/common.sh

@@ -248,7 +248,7 @@ function detect-master-from-kubeconfig() {
 
 # Sets KUBE_VERSION variable to the proper version number (e.g. "v1.0.6",
 # "v1.2.0-alpha.1.881+376438b69c7612") or a version' publication of the form
-# <bucket>/<version> (e.g. "release/stable",' "ci/latest-1").
+# <path>/<version> (e.g. "release/stable",' "ci/latest-1").
 #
 # See the docs on getting builds for more information about version
 # publication.
@@ -259,7 +259,12 @@ function detect-master-from-kubeconfig() {
 #   KUBE_VERSION
 function set_binary_version() {
   if [[ "${1}" =~ "/" ]]; then
-    KUBE_VERSION=$(gsutil cat gs://kubernetes-release/${1}.txt)
+    IFS='/' read -a path <<< "${1}"
+    if [[ "${path[0]}" == "release" ]]; then
+      KUBE_VERSION=$(gsutil cat "gs://kubernetes-release/${1}.txt")
+    else
+      KUBE_VERSION=$(gsutil cat "gs://kubernetes-release-dev/${1}.txt")
+    fi
   else
     KUBE_VERSION=${1}
   fi
@@ -287,8 +292,8 @@ function tars_from_version() {
     SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
     SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/kubernetes-salt.tar.gz"
   elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then
-    SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
-    SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${KUBE_VERSION}/kubernetes-salt.tar.gz"
+    SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release-dev/ci/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
+    SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release-dev/ci/${KUBE_VERSION}/kubernetes-salt.tar.gz"
   else
     echo "Version doesn't match regexp" >&2
     exit 1

cluster/gce/upgrade.sh

@@ -60,7 +60,7 @@ function usage() {
 
   release_stable=$(gsutil cat gs://kubernetes-release/release/stable.txt)
   release_latest=$(gsutil cat gs://kubernetes-release/release/latest.txt)
-  ci_latest=$(gsutil cat gs://kubernetes-release/ci/latest.txt)
+  ci_latest=$(gsutil cat gs://kubernetes-release-dev/ci/latest.txt)
 
   echo "Right now, versions are as follows:"
   echo "  release/stable: ${0} ${release_stable}"

docs/devel/getting-builds.md

@@ -30,9 +30,9 @@ Finally, you can just print the latest or stable version:
 You can also use the gsutil tool to explore the Google Cloud Storage release buckets. Here are some examples:
 
 ```sh
-gsutil cat gs://kubernetes-release/ci/latest.txt          # output the latest ci version number
-gsutil cat gs://kubernetes-release/ci/latest-green.txt    # output the latest ci version number that passed gce e2e
-gsutil ls gs://kubernetes-release/ci/v0.20.0-29-g29a55cc/ # list the contents of a ci release
+gsutil cat gs://kubernetes-release-dev/ci/latest.txt          # output the latest ci version number
+gsutil cat gs://kubernetes-release-dev/ci/latest-green.txt    # output the latest ci version number that passed gce e2e
+gsutil ls gs://kubernetes-release-dev/ci/v0.20.0-29-g29a55cc/ # list the contents of a ci release
 gsutil ls gs://kubernetes-release/release                 # list all official releases and rcs
 ```
 

hack/get-build.sh

@@ -23,6 +23,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/cluster/common.sh"
 
 declare -r KUBE_RELEASE_BUCKET_URL="https://storage.googleapis.com/kubernetes-release"
+declare -r KUBE_DEV_RELEASE_BUCKET_URL="https://storage.googleapis.com/kubernetes-release-dev"
 declare -r KUBE_TAR_NAME="kubernetes.tar.gz"
 
 usage() {
@@ -74,7 +75,7 @@ else
   if [[ ${KUBE_VERSION} =~ ${KUBE_RELEASE_VERSION_REGEX} ]]; then
     curl --fail -o "kubernetes-${KUBE_VERSION}.tar.gz" "${KUBE_RELEASE_BUCKET_URL}/release/${KUBE_VERSION}/${KUBE_TAR_NAME}"
   elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then
-    curl --fail -o "kubernetes-${KUBE_VERSION}.tar.gz" "${KUBE_RELEASE_BUCKET_URL}/ci/${KUBE_VERSION}/${KUBE_TAR_NAME}"
+    curl --fail -o "kubernetes-${KUBE_VERSION}.tar.gz" "${KUBE_DEV_RELEASE_BUCKET_URL}/ci/${KUBE_VERSION}/${KUBE_TAR_NAME}"
   else
     echo "Version doesn't match regexp" >&2
     exit 1

hack/jenkins/e2e-runner.sh

@@ -62,10 +62,15 @@ function fetch_published_version_tars() {
     clean_binaries
     local -r published_version="${1}"
     IFS='/' read -a varr <<< "${published_version}"
-    bucket="${varr[0]}"
-    build_version=$(gsutil cat gs://kubernetes-release/${published_version}.txt)
+    local -r path="${varr[0]}"
+    if [[ "${path}" == "release" ]]; then
+      local -r bucket="kubernetes-release"
+    else
+      local -r bucket="kubernetes-release-dev"
+    fi
+    build_version=$(gsutil cat "gs://${bucket}/${published_version}.txt")
     echo "Using published version $bucket/$build_version (from ${published_version})"
-    fetch_tars_from_gcs "${bucket}" "${build_version}"
+    fetch_tars_from_gcs "gs://${bucket}/${path}" "${build_version}"
     unpack_binaries
     # Set CLUSTER_API_VERSION for GKE CI
     export CLUSTER_API_VERSION=$(echo ${build_version} | cut -c 2-)
@@ -79,13 +84,10 @@ function clean_binaries() {
 }
 
 function fetch_tars_from_gcs() {
-    local -r bucket="${1}"
+    local -r gspath="${1}"
     local -r build_version="${2}"
-    echo "Pulling binaries from GCS; using server version ${bucket}/${build_version}."
-    gsutil -mq cp \
-        "gs://kubernetes-release/${bucket}/${build_version}/kubernetes.tar.gz" \
-        "gs://kubernetes-release/${bucket}/${build_version}/kubernetes-test.tar.gz" \
-        .
+    echo "Pulling binaries from GCS; using server version ${gspath}/${build_version}."
+    gsutil -mq cp "${gspath}/${build_version}/kubernetes.tar.gz" "${gspath}/${build_version}/kubernetes-test.tar.gz" .
 }
 
 function unpack_binaries() {