From 0b8538ddf58d396cde7e90c365e71b1442dfa426 Mon Sep 17 00:00:00 2001
From: Erick Fejta <fejta@google.com>
Date: Sat, 21 Jan 2017 11:59:30 -0800
Subject: [PATCH] Adding --build B, --stage, --extract E flags

---
 jenkins/e2e-image/Dockerfile    |   5 +-
 jenkins/e2e-image/e2e-runner.sh | 442 ++++----------------------------
 jobs/config.json                |   1 -
 kubetest/BUILD                  |   3 +
 kubetest/build.go               |  77 ++++++
 kubetest/e2e.go                 | 176 +++++--------
 kubetest/extract.go             | 368 ++++++++++++++++++++++++++
 kubetest/federation.go          |   6 +-
 kubetest/main.go                | 436 ++++++++++++++++++++++++++++---
 kubetest/none.go                |  18 +-
 kubetest/stage.go               |  79 ++++++
 kubetest/util.go                | 119 ++++++---
 12 files changed, 1142 insertions(+), 588 deletions(-)
 create mode 100644 kubetest/build.go
 create mode 100644 kubetest/extract.go
 create mode 100644 kubetest/stage.go

diff --git a/jenkins/e2e-image/Dockerfile b/jenkins/e2e-image/Dockerfile
index 8f548bf11b27..c84abce6c830 100644
--- a/jenkins/e2e-image/Dockerfile
+++ b/jenkins/e2e-image/Dockerfile
@@ -61,10 +61,9 @@ ADD ["e2e-runner.sh", \
     "kops-e2e-runner.sh", \
     "kubetest", \
     "https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/get-kube.sh", \
-    "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh", \
-    "https://raw.githubusercontent.com/kubernetes/kubernetes/master/third_party/forked/shell2junit/sh2ju.sh", \
+    "kubetest", \
     "/workspace/"]
-RUN ["chmod", "+x", "/workspace/get-kube.sh", "/workspace/sh2ju.sh"]
+RUN ["chmod", "+x", "/workspace/get-kube.sh"]
 WORKDIR "/workspace"
 
 ENTRYPOINT "/workspace/e2e-runner.sh"
diff --git a/jenkins/e2e-image/e2e-runner.sh b/jenkins/e2e-image/e2e-runner.sh
index a4d5883b9ae4..22a155578210 100755
--- a/jenkins/e2e-image/e2e-runner.sh
+++ b/jenkins/e2e-image/e2e-runner.sh
@@ -20,34 +20,10 @@ set -o nounset
 set -o pipefail
 set -o xtrace
 
-case "${KUBERNETES_PROVIDER}" in
-    gce|gke|kubemark)
-        if [[ -z "${PROJECT:-}" ]]; then
-          echo "ERROR: unset PROJECT" >&2
-          exit 1
-        fi
-        ;;
-esac
-
-# include shell2junit library
-sh2ju="$(dirname "${0}")/sh2ju.sh"
-if [[ -f "${sh2ju}" ]]; then
-  source "${sh2ju}"
-else
-  echo "TODO(fejta): stop pulling sh2ju.sh"
-  source <(curl -fsS --retry 3 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/third_party/forked/shell2junit/sh2ju.sh')
-fi
-
 # Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml
 ARTIFACTS=${WORKSPACE}/_artifacts
 mkdir -p ${ARTIFACTS}
 
-# E2E runner stages
-STAGE_PRE="PRE-SETUP"
-STAGE_SETUP="SETUP"
-STAGE_CLEANUP="CLEANUP"
-STAGE_KUBEMARK="KUBEMARK"
-
 : ${KUBE_GCS_RELEASE_BUCKET:="kubernetes-release"}
 : ${KUBE_GCS_DEV_RELEASE_BUCKET:="kubernetes-release-dev"}
 JENKINS_SOAK_PREFIX="gs://kubernetes-jenkins/soak/${JOB_NAME}"
@@ -55,347 +31,30 @@ JENKINS_SOAK_PREFIX="gs://kubernetes-jenkins/soak/${JOB_NAME}"
 # Explicitly set config path so staging gcloud (if installed) uses same path
 export CLOUDSDK_CONFIG="${WORKSPACE}/.config/gcloud"
 
-# record_command runs the command and records its output/error messages in junit format
-# it expects the first argument to be the class and the second to be the name of the command
-# Example:
-# record_command PRESETUP curltest curl google.com
-# record_command CLEANUP check false
-#
-# WARNING: Variable changes in the command will NOT be effective after record_command returns.
-#          This is because the command runs in subshell.
-function record_command() {
-    set +o xtrace
-    set +o nounset
-    set +o errexit
-
-    local class=$1
-    shift
-    local name=$1
-    shift
-    echo "Recording: ${class} ${name}"
-    echo "Running command: $@"
-    juLog -output="${ARTIFACTS}" -class="${class}" -name="${name}" "$@"
-
-    set -o nounset
-    set -o errexit
-    set -o xtrace
-}
-
-function running_in_docker() {
-    grep -q docker /proc/self/cgroup
-}
-
-# Sets KUBERNETES_RELEASE and KUBERNETES_RELEASE_URL to point to tarballs in the
-# local _output/gcs-stage directory.
-function set_release_vars_from_local_gcs_stage() {
-    local -r local_gcs_stage_path="${WORKSPACE}/_output/gcs-stage"
-    KUBERNETES_RELEASE_URL="file://${local_gcs_stage_path}"
-    KUBERNETES_RELEASE=$(ls "${local_gcs_stage_path}" | grep ^v.*$)
-    if [[ -z "${KUBERNETES_RELEASE}" ]]; then
-      echo "FAIL! version not found in ${local_gcs_stage_path}"
-      return 1
-    fi
-}
-
-# Use a published version like "ci/latest" (default), "release/latest",
-# "release/latest-1", or "release/stable".
-# TODO(ixdy): maybe this should be in get-kube.sh?
-function set_release_vars_from_gcs() {
-    local -r published_version="${1}"
-    IFS='/' read -a varr <<< "${published_version}"
-    local -r path="${varr[0]}"
-    if [[ "${path}" == "release" ]]; then
-      local -r bucket="${KUBE_GCS_RELEASE_BUCKET}"
-    else
-      local -r bucket="${KUBE_GCS_DEV_RELEASE_BUCKET}"
-    fi
-    KUBERNETES_RELEASE=$(gsutil cat "gs://${bucket}/${published_version}.txt")
-    KUBERNETES_RELEASE_URL="https://storage.googleapis.com/${bucket}/${path}"
-}
-
-function set_release_vars_from_gke_cluster_version() {
-    local -r server_version="$(gcloud ${CMD_GROUP:-} container get-server-config --project=${PROJECT} --zone=${ZONE}  --format='value(defaultClusterVersion)')"
-    # Use latest build of the server version's branch for test files.
-    set_release_vars_from_gcs "ci/latest-${server_version:0:3}"
-}
-
-function call_get_kube() {
-    local get_kube_sh="${WORKSPACE}/get-kube.sh"
-    if [[ ! -x "${get_kube_sh}" ]]; then
-      # If running outside docker (e.g. in soak tests) we may not have the
-      # script, so download it.
-      mkdir -p "${WORKSPACE}/_tmp/"
-      get_kube_sh="${WORKSPACE}/_tmp/get-kube.sh"
-      curl -fsSL --retry 3 --keepalive-time 2 https://get.k8s.io/ > "${get_kube_sh}"
-      chmod +x "${get_kube_sh}"
-    fi
-    export KUBERNETES_RELEASE
-    export KUBERNETES_RELEASE_URL
-    KUBERNETES_SKIP_CONFIRM=y KUBERNETES_SKIP_CREATE_CLUSTER=y KUBERNETES_DOWNLOAD_TESTS=y \
-      "${get_kube_sh}"
-    if [[ ! -x kubernetes/cluster/get-kube-binaries.sh ]]; then
-      # If the get-kube-binaries.sh script doesn't exist, assume this is an older
-      # release without it, and thus the tests haven't been downloaded yet.
-      # We'll have to download and extract them ourselves instead.
-      echo "Grabbing test binaries since cluster/get-kube-binaries.sh does not exist."
-      local -r test_tarball=kubernetes-test.tar.gz
-      curl -L "${KUBERNETES_RELEASE_URL:-https://storage.googleapis.com/kubernetes-release/release}/${KUBERNETES_RELEASE}/${test_tarball}" -o "${test_tarball}"
-      md5sum "${test_tarball}"
-      tar -xzf "${test_tarball}"
-    fi
-}
-
-# TODO(ihmccreery) I'm not sure if this is necesssary, with the workspace check
-# below.
-function clean_binaries() {
-    echo "Cleaning up binaries."
-    rm -rf kubernetes*
-}
-
-function get_latest_docker_release() {
-  # Typical Docker release versions are like v1.11.2-rc1, v1.11.2, and etc.
-  local -r version_re='.*\"tag_name\":[[:space:]]+\"v([0-9\.r|c-]+)\",.*'
-  local -r releases="$(curl -fsSL --retry 3 https://api.github.com/repos/docker/docker/releases)"
-  # The GitHub API returns releases in descending order of creation time so the
-  # first one is always the latest.
-  # TODO: if we can install `jq` on the Jenkins nodes, we won't have to craft
-  # regular expressions here.
-  while read -r line; do
-    if [[ "${line}" =~ ${version_re} ]]; then
-      echo "${BASH_REMATCH[1]}"
-      return
-    fi
-  done <<< "${releases}"
-  echo "Failed to determine the latest Docker release."
-  exit 1
-}
-
-function install_google_cloud_sdk_tarball() {
-    local -r tarball=$1
-    local -r install_dir=$2
-
-    if running_in_docker; then
-        # Delete any previously installed SDK to prevent weirdly overlapping
-        # installs from fighting each other.
-        # TODO(rmmh): should just do a components update instead?
-        rm -rf "${install_dir}/google-cloud-sdk"
-    fi
-
-    mkdir -p "${install_dir}"
-    tar xzf "${tarball}" -C "${install_dir}"
-    export CLOUDSDK_CORE_DISABLE_PROMPTS=1
-    record_command "${STAGE_PRE}" "install_gcloud" "${install_dir}/google-cloud-sdk/install.sh" --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false
-    export PATH=${install_dir}/google-cloud-sdk/bin:${PATH}
-    gcloud components install alpha
-    gcloud components install beta
-    gcloud info
-}
-
-# Sets release vars using GCI image builtin k8s version.
-# If JENKINS_GCI_PATCH_K8S is set, uses the latest CI build on the same branch
-# instead.
-# Assumes: JENKINS_GCI_HEAD_IMAGE_FAMILY and KUBE_GCE_MASTER_IMAGE
-function set_release_vars_from_gci_builtin_version() {
-    if ! [[ "${JENKINS_USE_GCI_VERSION:-}" =~ ^[yY]$ ]]; then
-        echo "JENKINS_USE_GCI_VERSION must be set."
-        exit 1
-    fi
-    if [[ -z "${JENKINS_GCI_HEAD_IMAGE_FAMILY:-}" ]] || [[ -z "${KUBE_GCE_MASTER_IMAGE:-}" ]]; then
-        echo "JENKINS_GCI_HEAD_IMAGE_FAMILY and KUBE_GCE_MASTER_IMAGE must both be set."
-        exit 1
-    fi
-    local -r gci_k8s_version="$(gsutil cat gs://container-vm-image-staging/k8s-version-map/${KUBE_GCE_MASTER_IMAGE})"
-    if [[ "${JENKINS_GCI_PATCH_K8S}" =~ ^[yY]$ ]]; then
-        # We always want to test against the builtin k8s version, but occationally
-        # the builtin version has known bugs that keep our tests red. In those
-        # cases, we use the latest CI build on the same branch instead.
-        set_release_vars_from_gcs "ci/latest-${gci_k8s_version:0:3}"
-    else
-        KUBERNETES_RELEASE="v${gci_k8s_version}"
-        # Use the default KUBERNETES_RELEASE_URL.
-    fi
-}
-
-# Specific settings for tests that use GCI HEAD images. I.e., if your test is
-# using a public/released GCI image, you don't want to call this function.
-# Assumes: JENKINS_GCI_HEAD_IMAGE_FAMILY
-function setup_gci_vars() {
-    local -r gci_staging_project=container-vm-image-staging
-    local -r image_name="$(gcloud compute images describe-from-family ${JENKINS_GCI_HEAD_IMAGE_FAMILY} --project=${gci_staging_project} --format='value(name)')"
-
-    export KUBE_GCE_MASTER_PROJECT="${gci_staging_project}"
-    export KUBE_GCE_MASTER_IMAGE="${image_name}"
-    export KUBE_MASTER_OS_DISTRIBUTION="gci"
-
-    export KUBE_GCE_NODE_PROJECT="${gci_staging_project}"
-    export KUBE_GCE_NODE_IMAGE="${image_name}"
-    export KUBE_NODE_OS_DISTRIBUTION="gci"
-
-    # These will be included in started.json in the metadata dict.
-    # See upload-to-gcs.sh for more details.
-    export BUILD_METADATA_GCE_MASTER_IMAGE="${KUBE_GCE_MASTER_IMAGE}"
-    export BUILD_METADATA_GCE_NODE_IMAGE="${KUBE_GCE_NODE_IMAGE}"
-
-    # For backward compatibility (Older versions of Kubernetes don't understand
-    # KUBE_MASTER_OS_DISTRIBUTION or KUBE_NODE_OS_DISTRIBUTION. Only KUBE_OS_DISTRIBUTION can be
-    # used for them.)
-    export KUBE_OS_DISTRIBUTION="gci"
-
-    if [[ "${JENKINS_GCI_HEAD_IMAGE_FAMILY}" == "gci-canary-test" ]]; then
-        # The family "gci-canary-test" is reserved for a special type of GCI images
-        # that are used to continuously validate Docker releases.
-        export KUBE_GCI_DOCKER_VERSION="$(get_latest_docker_release)"
-    fi
-}
-
-### Pre Set Up ###
-if running_in_docker; then
-    record_command "${STAGE_PRE}" "download_gcloud" curl -fsSL --retry 3 --keepalive-time 2 -o "${WORKSPACE}/google-cloud-sdk.tar.gz" 'https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz'
-    install_google_cloud_sdk_tarball "${WORKSPACE}/google-cloud-sdk.tar.gz" /
-    if [[ "${KUBERNETES_PROVIDER}" == 'aws' ]]; then
-        pip install awscli
-    fi
-fi
-
-if [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then
-  gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
-fi
-
-# Install gcloud from a custom path if provided. Used to test GKE with gcloud
-# at HEAD, release candidate.
-# TODO: figure out how to avoid installing the cloud sdk twice if run inside Docker.
-if [[ -n "${CLOUDSDK_BUCKET:-}" ]]; then
-    # Retry the download a few times to mitigate transient server errors and
-    # race conditions where the bucket contents change under us as we download.
-    for n in {1..3}; do
-        gsutil -mq cp -r "${CLOUDSDK_BUCKET}" ~ && break || sleep 1
-        # Delete any temporary files from the download so that we start from
-        # scratch when we retry.
-        rm -rf ~/.gsutil
-    done
-    rm -rf ~/repo ~/cloudsdk
-    mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo
-    export CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL=file://${HOME}/repo/components-2.json
-    install_google_cloud_sdk_tarball ~/repo/google-cloud-sdk.tar.gz ~/cloudsdk
-
-    # Just in case the new gcloud stores credentials differently, re-activate
-    # credentials.
-    if [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then
-      gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
-    fi
-fi
-
-# Specific settings for tests that use GCI HEAD images. I.e., if your test is
-# using a public/released GCI image, you don't want to set this variable or call
-# `setup_gci_vars`.
-if [[ -n "${JENKINS_GCI_HEAD_IMAGE_FAMILY:-}" ]]; then
-  setup_gci_vars
-fi
-
 echo "--------------------------------------------------------------------------------"
 echo "Test Environment:"
 printenv | sort
 echo "--------------------------------------------------------------------------------"
 
-# Set this var instead of exiting-- we must do the cluster teardown step. We'll
-# return this at the very end.
-EXIT_CODE=0
-
-# We get the Kubernetes tarballs unless we are going to use old ones
-if [[ "${JENKINS_USE_EXISTING_BINARIES:-}" =~ ^[yY]$ ]]; then
-  echo "Using existing binaries; not cleaning, fetching, or unpacking new ones."
-else
-  clean_binaries
-
-  if [[ "${JENKINS_SOAK_MODE:-}" == "y" && ${E2E_UP:-} != "true" ]]; then
-    # We are restoring the cluster, copy state from gcs
-    # TODO(fejta): auto-detect and recover from deployment failures
-    mkdir -p "${HOME}/.kube"
-    gsutil cp "${JENKINS_SOAK_PREFIX}/kube-config" "${HOME}/.kube/config"
-    export KUBERNETES_RELEASE=$(gsutil cat "${JENKINS_SOAK_PREFIX}/release.txt")
-    export KUBERNETES_RELEASE_URL=$(gsutil cat "${JENKINS_SOAK_PREFIX}/release-url.txt")
-    export CLUSTER_API_VERSION=$(echo "${KUBERNETES_RELEASE}" | cut -c 2-)  # for GKE CI
-  elif [[ "${JENKINS_USE_LOCAL_BINARIES:-}" =~ ^[yY]$ ]]; then
-    set_release_vars_from_local_gcs_stage
-  elif [[ "${JENKINS_USE_SERVER_VERSION:-}" =~ ^[yY]$ ]]; then
-    # This is for test, staging, and prod jobs on GKE, where we want to
-    # test what's running in GKE by default rather than some CI build.
-    set_release_vars_from_gke_cluster_version
-  elif [[ "${JENKINS_USE_GCI_VERSION:-}" =~ ^[yY]$ ]]; then
-    # Use GCI image builtin version. Needed for GCI release qual tests.
-    set_release_vars_from_gci_builtin_version
-  else
-    # use JENKINS_PUBLISHED_VERSION, default to 'ci/latest', since that's
-    # usually what we're testing.
-    set_release_vars_from_gcs "${JENKINS_PUBLISHED_VERSION:-ci/latest}"
-    # Needed for GKE CI.
-    export CLUSTER_API_VERSION=$(echo "${KUBERNETES_RELEASE}" | cut -c 2-)
-  fi
-
-  call_get_kube
-fi
-
-# Copy GCE keys so we don't keep cycling them.
-# To set this up, you must know the <project>, <zone>, and <instance>
-# on which your jenkins jobs are running. Then do:
-#
-# # SSH from your computer into the instance.
-# $ gcloud compute ssh --project="<prj>" ssh --zone="<zone>" <instance>
-#
-# # Generate a key by ssh'ing from the instance into itself, then exit.
-# $ gcloud compute ssh --project="<prj>" ssh --zone="<zone>" <instance>
-# $ ^D
-#
-# # Copy the keys to the desired location (e.g. /var/lib/jenkins/gce_keys/).
-# $ sudo mkdir -p /var/lib/jenkins/gce_keys/
-# $ sudo cp ~/.ssh/google_compute_engine /var/lib/jenkins/gce_keys/
-# $ sudo cp ~/.ssh/google_compute_engine.pub /var/lib/jenkins/gce_keys/
-#
-# # Move the permissions for the keys to Jenkins.
-# $ sudo chown -R jenkins /var/lib/jenkins/gce_keys/
-# $ sudo chgrp -R jenkins /var/lib/jenkins/gce_keys/
-case "${KUBERNETES_PROVIDER}" in
-    gce|gke|kubemark)
-        echo 'Checking existence of private ssh key'
-        gce_key="${WORKSPACE}/.ssh/google_compute_engine"
-        if [[ ! -f "${gce_key}" || ! -f "${gce_key}.pub" ]]; then
-            echo 'google_compute_engine ssh key missing!'
-            exit 1
-        fi
-        echo "Checking presence of public key in ${PROJECT}"
-        if ! gcloud compute --project="${PROJECT}" project-info describe |
-             grep "$(cat "${gce_key}.pub")" >/dev/null; then
-            echo 'Uploading public ssh key to project metadata...'
-            gcloud compute --project="${PROJECT}" config-ssh
-        fi
-        ;;
-    default)
-        echo "Not copying ssh keys for ${KUBERNETES_PROVIDER}"
-        ;;
-esac
+# When run inside Docker, we need to make sure all files are world-readable
+# (since they will be owned by root on the host).
+trap "chmod -R o+r '${ARTIFACTS}'" EXIT SIGINT SIGTERM
+export E2E_REPORT_DIR=${ARTIFACTS}
 
+e2e_go_args=( \
+  -v \
+  --dump="${ARTIFACTS}" \
+)
 
 # Allow download & unpack of alternate version of tests, for cross-version & upgrade testing.
 #
-# JENKINS_PUBLISHED_SKEW_VERSION downloads an alternate version of Kubernetes
-# for testing, moving the old one to kubernetes_old.
+# JENKINS_PUBLISHED_SKEW_VERSION adds a second --extract before the other one.
+# The JENKINS_PUBLISHED_SKEW_VERSION extracts to kubernetes_skew.
+# The JENKINS_PUBLISHED_VERSION extracts to kubernetes.
 #
-# E2E_UPGRADE_TEST=true triggers a run of the e2e tests, to do something like
-# upgrade the cluster, before the main test run.  It uses
-# GINKGO_UPGRADE_TESTS_ARGS for the test run.
-#
-# JENKINS_USE_SKEW_TESTS=true will run tests from the skewed version rather
-# than the original version.
+# For upgrades, PUBLISHED_SKEW should be a new release than PUBLISHED.
 if [[ -n "${JENKINS_PUBLISHED_SKEW_VERSION:-}" ]]; then
-  mv kubernetes kubernetes_orig
-  (
-      # Subshell so we don't override KUBERNETES_RELEASE
-      set_release_vars_from_gcs "${JENKINS_PUBLISHED_SKEW_VERSION}"
-      call_get_kube
-  )
-  mv kubernetes kubernetes_skew
-  mv kubernetes_orig kubernetes
-  export BUILD_METADATA_KUBERNETES_SKEW_VERSION=$(cat kubernetes_skew/version || true)
+  e2e_go_args+=(--extract="${JENKINS_PUBLISHED_SKEW_VERSION}")
   if [[ "${JENKINS_USE_SKEW_TESTS:-}" != "true" ]]; then
     # Append kubectl-path of skewed kubectl to test args, since we always
     # want that to use the skewed kubectl version:
@@ -404,34 +63,44 @@ if [[ -n "${JENKINS_PUBLISHED_SKEW_VERSION:-}" ]]; then
     #  - for client skew tests, we want to use the skewed kubectl
     #    (that's what we're testing).
     GINKGO_TEST_ARGS="${GINKGO_TEST_ARGS:-} --kubectl-path=$(pwd)/kubernetes_skew/cluster/kubectl.sh"
+  else
+    e2e_go_args+=(--skew)  # Get kubectl as well as test code from kubernetes_skew
   fi
 fi
 
-cd kubernetes
-
-source "$(dirname "${0}")/upload-to-gcs.sh"
-version=$(find_version)  # required by print_started
-print_started | jq '.metadata? + {version, "job-version"}' > "${ARTIFACTS}/metadata.json"
 
-if [[ -n "${PRIORITY_PATH:-}" ]]; then
-  export PATH="${PRIORITY_PATH}:${PATH}"
+# We get the Kubernetes tarballs unless we are going to use old ones
+if [[ "${JENKINS_USE_EXISTING_BINARIES:-}" =~ ^[yY]$ ]]; then
+  echo "Using existing binaries; not cleaning, fetching, or unpacking new ones."
+elif [[ "${JENKINS_USE_LOCAL_BINARIES:-}" =~ ^[yY]$ ]]; then
+  e2e_go_args+=(--extract="local")
+elif [[ "${JENKINS_USE_SERVER_VERSION:-}" =~ ^[yY]$ ]]; then
+  # This is for test, staging, and prod jobs on GKE, where we want to
+  # test what's running in GKE by default rather than some CI build.
+  e2e_go_args+=(--extract="gke")
+elif [[ "${JENKINS_USE_GCI_VERSION:-}" =~ ^[yY]$ ]]; then
+  # Use GCI image builtin version. Needed for GCI release qual tests.
+  e2e_go_args+=(--extract="gci/${JENKINS_GCI_HEAD_IMAGE_FAMILY}")
+else
+  # use JENKINS_PUBLISHED_VERSION, default to 'ci/latest', since that's
+  # usually what we're testing.
+  e2e_go_args+=(--extract="${JENKINS_PUBLISHED_VERSION:-ci/latest}")
 fi
 
-# When run inside Docker, we need to make sure all files are world-readable
-# (since they will be owned by root on the host).
-trap "chmod -R o+r '${ARTIFACTS}'" EXIT SIGINT SIGTERM
-export E2E_REPORT_DIR=${ARTIFACTS}
-
-e2e_go_args=( \
-  -v \
-  --dump="${ARTIFACTS}" \
-)
+if [[ "${JENKINS_SOAK_MODE:-}" == "y" ]]; then
+  # In soak mode we sync cluster state to gcs.
+  # If we --up a cluster, we save the kubecfg and version info to gcs.
+  # Otherwise we load kubecfg and version info from gcs.
+  #e2e_go_args+=(--save="gs://${JENKINS_SOAK_PREFIX}")
+  # TODO FIX ME DO NOT SUBMIT AAAAAAAAAAAAAAAAAAA
+  e2e_go_args+=(--save="gs://fejternetes/soak")
+fi
 
 
 if [[ "${FAIL_ON_GCP_RESOURCE_LEAK:-true}" == "true" ]]; then
   case "${KUBERNETES_PROVIDER}" in
     gce|gke)
-      e2e_go_args+=(--check_leaked_resources)
+      e2e_go_args+=(--check-leaked-resources)
       ;;
   esac
 fi
@@ -442,7 +111,7 @@ if [[ "${FEDERATION:-}" == "true" ]]; then
 fi
 
 if [[ "${E2E_UP:-}" == "true" ]] || [[ "${FEDERATION_UP:-}" == "true" ]]; then
-  e2e_go_args+=(--up --ctl="version --match-server-version=false")
+  e2e_go_args+=(--up)
 fi
 
 if [[ "${E2E_DOWN:-}" == "true" ]] || [[ "${FEDERATION_DOWN:-}" == "true" ]]; then
@@ -451,10 +120,9 @@ fi
 
 if [[ "${FEDERATION_UP:-}" == "true" ]] || [[ "${FEDERATION_DOWN:-}" == "true" ]]; then
   e2e_go_args+=(--federation)
-fi
-
-if [[ -z "${FEDERATION_CLUSTERS:-}" ]]; then
-  e2e_go_args+=("--deployment=none")
+  if [[ -z "${FEDERATION_CLUSTERS:-}" ]]; then
+    e2e_go_args+=("--deployment=none")
+  fi
 fi
 
 if [[ "${E2E_TEST:-}" == "true" ]]; then
@@ -464,18 +132,13 @@ if [[ "${E2E_TEST:-}" == "true" ]]; then
   fi
 fi
 
-# Optionally run tests from the version in  kubernetes_skew
-if [[ "${JENKINS_USE_SKEW_TESTS:-}" == "true" ]]; then
-  e2e_go_args+=(--skew)
-fi
-
 # Optionally run upgrade tests before other tests.
 if [[ "${E2E_UPGRADE_TEST:-}" == "true" ]]; then
   e2e_go_args+=(--upgrade_args="${GINKGO_UPGRADE_TEST_ARGS}")
 fi
 
 if [[ "${USE_KUBEMARK:-}" == "true" ]]; then
-  e2e_go_args+=("--kubemark=true")
+  e2e_go_args+=(--kubemark=true)
 fi
 
 if [[ "${CHARTS_TEST:-}" == "true" ]]; then
@@ -483,19 +146,16 @@ if [[ "${CHARTS_TEST:-}" == "true" ]]; then
 fi
 
 if [[ -n "${KUBEKINS_TIMEOUT:-}" ]]; then
-  e2e_go_args+=("--timeout=${KUBEKINS_TIMEOUT}")
+  e2e_go_args+=(--timeout="${KUBEKINS_TIMEOUT}")
 fi
 
 if [[ -n "${E2E_PUBLISH_PATH:-}" ]]; then
   e2e_go_args+=(--publish="${E2E_PUBLISH_PATH}")
 fi
 
-
-"$(dirname "${0}")/kubetest" ${E2E_OPT:-} "${e2e_go_args[@]}"
-
-if [[ "${JENKINS_SOAK_MODE:-}" == "y" && ${E2E_UP:-} == "true" ]]; then
-  # We deployed a cluster, save state to gcs
-  gsutil cp -a project-private "${HOME}/.kube/config" "${JENKINS_SOAK_PREFIX}/kube-config"
-  echo ${KUBERNETES_RELEASE} | gsutil cp - "${JENKINS_SOAK_PREFIX}/release.txt"
-  echo ${KUBERNETES_RELEASE_URL} | gsutil cp - "${JENKINS_SOAK_PREFIX}/release-url.txt"
+if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" ]]; then
+  # Use plaintext version file packaged with kubernetes.tar.gz
+  e2e_go_args+=(--publish="gs://${KUBE_GCS_DEV_RELEASE_BUCKET}/ci/latest-green.txt")
 fi
+
+./kubetest ${E2E_OPT:-} "${e2e_go_args[@]}"
diff --git a/jobs/config.json b/jobs/config.json
index df830fa0b306..df76ecc8be29 100644
--- a/jobs/config.json
+++ b/jobs/config.json
@@ -3215,5 +3215,4 @@
     "--env-file=jobs/ci-kubernetes-e2e-gke-latest-upgrade-cluster.env"
   ]
 }
-
 }
diff --git a/kubetest/BUILD b/kubetest/BUILD
index b40af172b2b6..8307e7dde5bf 100644
--- a/kubetest/BUILD
+++ b/kubetest/BUILD
@@ -19,11 +19,14 @@ go_library(
     srcs = [
         "anywhere.go",
         "bash.go",
+        "build.go",
         "e2e.go",
+        "extract.go",
         "federation.go",
         "kops.go",
         "main.go",
         "none.go",
+        "stage.go",
         "util.go",
     ],
     tags = ["automanaged"],
diff --git a/kubetest/build.go b/kubetest/build.go
new file mode 100644
index 000000000000..dc22d3ccc77a
--- /dev/null
+++ b/kubetest/build.go
@@ -0,0 +1,77 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os/exec"
+)
+
+const (
+	buildDefault = "quick"
+)
+
+type buildStrategy string
+
+// Support both --build and --build=foo
+func (b *buildStrategy) IsBoolFlag() bool {
+	return true
+}
+
+// Return b as a string
+func (b *buildStrategy) String() string {
+	return string(*b)
+}
+
+// Set to --build=B or buildDefault if just --build
+func (b *buildStrategy) Set(value string) error {
+	if value == "true" { // just --build, choose default
+		value = buildDefault
+	}
+	switch value {
+	case "bazel", "quick", "release":
+		*b = buildStrategy(value)
+		return nil
+	}
+	return fmt.Errorf("Bad build strategy: %v (use: bash, quick, release)", value)
+}
+
+// True when this kubetest invocation wants to build a release
+func (b *buildStrategy) Enabled() bool {
+	return *b != ""
+}
+
+// Build kubernetes according to specified strategy.
+// This may be a bazel, quick or full release build depending on --build=B.
+func (b *buildStrategy) Build() error {
+	var target string
+	switch *b {
+	case "bazel":
+		target = "bazel-build"
+	case "quick":
+		target = "quick-release"
+	case "release":
+		target = "release"
+	default:
+		return fmt.Errorf("Unknown build strategy: %v", b)
+	}
+
+	// TODO(fejta): FIX ME
+	// The build-release script needs stdin to ask the user whether
+	// it's OK to download the docker image.
+	return finishRunning(exec.Command("make", target))
+}
diff --git a/kubetest/e2e.go b/kubetest/e2e.go
index d9b333384587..bfb8d290e7c4 100644
--- a/kubetest/e2e.go
+++ b/kubetest/e2e.go
@@ -28,25 +28,8 @@ import (
 	"time"
 )
 
-func appendError(errs []error, err error) []error {
-	if err != nil {
-		return append(errs, err)
-	}
-	return errs
-}
-
-func run(deploy deployer, fedDeploy federationDeployer) error {
-	if *dump != "" {
-		defer writeXML(time.Now())
-	}
-
-	if *build {
-		if err := xmlWrap("Build", Build); err != nil {
-			return fmt.Errorf("error building: %s", err)
-		}
-	}
-
-	if *checkVersionSkew {
+func run(deploy deployer, o options) error {
+	if o.checkSkew {
 		os.Setenv("KUBECTL", "./cluster/kubectl.sh --match-server-version")
 	} else {
 		os.Setenv("KUBECTL", "./cluster/kubectl.sh")
@@ -55,7 +38,7 @@ func run(deploy deployer, fedDeploy federationDeployer) error {
 	// force having batch/v2alpha1 always on for e2e tests
 	os.Setenv("KUBE_RUNTIME_CONFIG", "batch/v2alpha1=true")
 
-	if *up {
+	if o.up {
 		if err := xmlWrap("TearDown Previous", deploy.Down); err != nil {
 			return fmt.Errorf("error tearing down previous cluster: %s", err)
 		}
@@ -77,17 +60,17 @@ func run(deploy deployer, fedDeploy federationDeployer) error {
 		afterResources  []byte
 	)
 
-	if *checkLeakedResources {
+	if o.checkLeaks {
 		errs = appendError(errs, xmlWrap("ListResources Before", func() error {
 			beforeResources, err = ListResources()
 			return err
 		}))
 	}
 
-	if *up {
+	if o.up {
 		// If we tried to bring the cluster up, make a courtesy
 		// attempt to bring it down so we're not leaving resources around.
-		if *down {
+		if o.down {
 			defer xmlWrap("Deferred TearDown", func() error {
 				if !downDone {
 					return deploy.Down()
@@ -97,10 +80,10 @@ func run(deploy deployer, fedDeploy federationDeployer) error {
 			// Deferred statements are executed in last-in-first-out order, so
 			// federation down defer must appear after the cluster teardown in
 			// order to execute that before cluster teardown.
-			if *federation {
+			if o.federation {
 				defer xmlWrap("Deferred Federation TearDown", func() error {
 					if !federationDownDone {
-						return fedDeploy.Down()
+						return FedDown()
 					}
 					return nil
 				})
@@ -108,79 +91,85 @@ func run(deploy deployer, fedDeploy federationDeployer) error {
 		}
 		// Start the cluster using this version.
 		if err := xmlWrap("Up", deploy.Up); err != nil {
-			if *dump != "" {
+			if o.dump != "" {
 				xmlWrap("DumpClusterLogs", func() error {
-					return DumpClusterLogs(*dump)
+					return DumpClusterLogs(o.dump)
 				})
 			}
 			return fmt.Errorf("starting e2e cluster: %s", err)
 		}
-		if *federation {
-			if err := xmlWrap("Federation Up", fedDeploy.Up); err != nil {
+		if o.federation {
+			if err := xmlWrap("Federation Up", FedUp); err != nil {
 				// TODO: Dump federation related logs.
 				return fmt.Errorf("error starting federation: %s", err)
 			}
 		}
-		if *dump != "" {
-			errs = appendError(errs, xmlWrap("list nodes", listNodes))
+		if o.dump != "" {
+			errs = appendError(errs, xmlWrap("list nodes", func() error {
+				return listNodes(o.dump)
+			}))
 		}
 	}
 
-	if *checkLeakedResources {
+	if o.checkLeaks {
 		errs = appendError(errs, xmlWrap("ListResources Up", func() error {
 			upResources, err = ListResources()
 			return err
 		}))
 	}
 
-	if *upgradeArgs != "" {
+	if o.upgradeArgs != "" {
 		errs = appendError(errs, xmlWrap("UpgradeTest", func() error {
-			return UpgradeTest(*upgradeArgs)
+			return UpgradeTest(o.upgradeArgs, o.checkSkew)
 		}))
 	}
 
-	if *test {
+	if o.test {
 		errs = appendError(errs, xmlWrap("get kubeconfig", deploy.SetupKubecfg))
 		errs = appendError(errs, xmlWrap("kubectl version", func() error {
 			return finishRunning(exec.Command("./cluster/kubectl.sh", "version", "--match-server-version=false"))
 		}))
-		if *skewTests {
-			errs = appendError(errs, xmlWrap("SkewTest", SkewTest))
+		if o.skew {
+			errs = appendError(errs, xmlWrap("SkewTest", func() error {
+				return SkewTest(o.testArgs, o.checkSkew)
+			}))
 		} else {
 			if err := xmlWrap("IsUp", deploy.IsUp); err != nil {
 				errs = appendError(errs, err)
 			} else {
-				errs = appendError(errs, xmlWrap("Test", Test))
+				errs = appendError(errs, xmlWrap("Test", func() error {
+					return Test(o.testArgs)
+				}))
 			}
 		}
 	}
 
-	if *kubemark {
+	if o.kubemark {
 		errs = appendError(errs, xmlWrap("Kubemark", KubemarkTest))
 	}
 
-	if *chartTests {
+	if o.charts {
 		errs = appendError(errs, xmlWrap("Helm Charts", ChartsTest))
 	}
 
-	if len(errs) > 0 && *dump != "" {
+	if len(errs) > 0 && o.dump != "" {
 		errs = appendError(errs, xmlWrap("DumpClusterLogs", func() error {
-			return DumpClusterLogs(*dump)
+			return DumpClusterLogs(o.dump)
 		}))
 	}
 
-	if *checkLeakedResources {
+	if o.checkLeaks {
 		errs = appendError(errs, xmlWrap("ListResources Down", func() error {
 			downResources, err = ListResources()
 			return err
 		}))
 	}
 
-	if *down {
-		if *federation {
+	if o.down {
+		if o.federation {
 			errs = appendError(errs, xmlWrap("Federation TearDown", func() error {
 				if !federationDownDone {
-					err := fedDeploy.Down()
+					err := FedDown()
 					if err != nil {
 						return err
 					}
@@ -201,7 +190,7 @@ func run(deploy deployer, fedDeploy federationDeployer) error {
 		}))
 	}
 
-	if *checkLeakedResources {
+	if o.checkLeaks {
 		log.Print("Sleeping for 30 seconds...") // Wait for eventually consistent listing
 		time.Sleep(30 * time.Second)
 		if err := xmlWrap("ListResources After", func() error {
@@ -211,19 +200,19 @@ func run(deploy deployer, fedDeploy federationDeployer) error {
 			errs = append(errs, err)
 		} else {
 			errs = appendError(errs, xmlWrap("DiffResources", func() error {
-				return DiffResources(beforeResources, upResources, downResources, afterResources, *dump)
+				return DiffResources(beforeResources, upResources, downResources, afterResources, o.dump)
 			}))
 		}
 	}
-	if len(errs) == 0 && *publish != "" {
+	if len(errs) == 0 && o.publish != "" {
 		errs = appendError(errs, xmlWrap("Publish version", func() error {
 			// Use plaintext version file packaged with kubernetes.tar.gz
 			if v, err := ioutil.ReadFile("version"); err != nil {
 				return err
 			} else {
-				log.Printf("Set %s version to %s", *publish, string(v))
+				log.Printf("Set %s version to %s", o.publish, string(v))
 			}
-			return finishRunning(exec.Command("gsutil", "cp", "version", *publish))
+			return finishRunning(exec.Command("gsutil", "cp", "version", o.publish))
 		}))
 	}
 
@@ -233,16 +222,15 @@ func run(deploy deployer, fedDeploy federationDeployer) error {
 	return nil
 }
 
-func listNodes() error {
-	cmd := exec.Command("./cluster/kubectl.sh", "--match-server-version=false", "get", "nodes", "-oyaml")
-	b, err := cmd.CombinedOutput()
-	if *verbose {
+func listNodes(dump string) error {
+	b, err := combinedOutput(exec.Command("./cluster/kubectl.sh", "--match-server-version=false", "get", "nodes", "-oyaml"))
+	if verbose {
 		log.Printf("kubectl get nodes:\n%s", string(b))
 	}
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(filepath.Join(*dump, "nodes.yaml"), b, 0644)
+	return ioutil.WriteFile(filepath.Join(dump, "nodes.yaml"), b, 0644)
 }
 
 func DiffResources(before, clusterUp, clusterDown, after []byte, location string) error {
@@ -275,7 +263,7 @@ func DiffResources(before, clusterUp, clusterDown, after []byte, location string
 	}
 
 	cmd := exec.Command("diff", "-sw", "-U0", "-F^\\[.*\\]$", bp, ap)
-	if *verbose {
+	if verbose {
 		cmd.Stderr = os.Stderr
 	}
 	stdout, cerr := cmd.Output()
@@ -311,7 +299,7 @@ func DiffResources(before, clusterUp, clusterDown, after []byte, location string
 func ListResources() ([]byte, error) {
 	log.Printf("Listing resources...")
 	cmd := exec.Command("./cluster/gce/list-resources.sh")
-	if *verbose {
+	if verbose {
 		cmd.Stderr = os.Stderr
 	}
 	stdout, err := cmd.Output()
@@ -321,43 +309,6 @@ func ListResources() ([]byte, error) {
 	return stdout, nil
 }
 
-func Build() error {
-	// The build-release script needs stdin to ask the user whether
-	// it's OK to download the docker image.
-	cmd := exec.Command("make", "quick-release")
-	cmd.Stdin = os.Stdin
-	if err := finishRunning(cmd); err != nil {
-		return fmt.Errorf("error building kubernetes: %v", err)
-	}
-	return nil
-}
-
-type deployer interface {
-	Up() error
-	IsUp() error
-	SetupKubecfg() error
-	Down() error
-}
-
-func getDeployer() (deployer, error) {
-	switch *deployment {
-	case "none":
-		return none{}, nil
-	case "bash":
-		return bash{}, nil
-	case "kops":
-		return NewKops()
-	case "kubernetes-anywhere":
-		return NewKubernetesAnywhere()
-	default:
-		return nil, fmt.Errorf("Unknown deployment strategy %q", *deployment)
-	}
-}
-
-func getFederationDeployer() (federationDeployer, error) {
-	return federationDeployer{}, nil
-}
-
 func clusterSize(deploy deployer) (int, error) {
 	if err := deploy.SetupKubecfg(); err != nil {
 		return -1, err
@@ -372,20 +323,20 @@ func clusterSize(deploy deployer) (int, error) {
 	return len(strings.Split(stdout, "\n")), nil
 }
 
-// CommandError will provide stderr output (if available) from structured
+// commandError will provide stderr output (if available) from structured
 // exit errors
-type CommandError struct {
+type commandError struct {
 	err error
 }
 
-func WrapError(err error) *CommandError {
+func WrapError(err error) *commandError {
 	if err == nil {
 		return nil
 	}
-	return &CommandError{err: err}
+	return &commandError{err: err}
 }
 
-func (e *CommandError) Error() string {
+func (e *commandError) Error() string {
 	if e == nil {
 		return ""
 	}
@@ -510,7 +461,8 @@ func chdirSkew() (string, error) {
 	return old, nil
 }
 
-func UpgradeTest(args string) error {
+func UpgradeTest(args string, checkSkew bool) error {
+	// TOOD(fejta): fix this
 	old, err := chdirSkew()
 	if err != nil {
 		return err
@@ -527,11 +479,11 @@ func UpgradeTest(args string) error {
 		"go", "run", "./hack/e2e.go",
 		"--test",
 		"--test_args="+args,
-		fmt.Sprintf("--v=%t", *verbose),
-		fmt.Sprintf("--check_version_skew=%t", *checkVersionSkew)))
+		fmt.Sprintf("--v=%t", verbose),
+		fmt.Sprintf("--check-version-skew=%t", checkSkew)))
 }
 
-func SkewTest() error {
+func SkewTest(args string, checkSkew bool) error {
 	old, err := chdirSkew()
 	if err != nil {
 		return err
@@ -540,19 +492,19 @@ func SkewTest() error {
 	return finishRunning(exec.Command(
 		"go", "run", "./hack/e2e.go",
 		"--test",
-		"--test_args="+*testArgs,
-		fmt.Sprintf("--v=%t", *verbose),
-		fmt.Sprintf("--check_version_skew=%t", *checkVersionSkew)))
+		"--test_args="+args,
+		fmt.Sprintf("--v=%t", verbose),
+		fmt.Sprintf("--check_version_skew=%t", checkSkew)))
 }
 
-func Test() error {
+func Test(testArgs string) error {
 	// TODO(fejta): add a --federated or something similar
 	if os.Getenv("FEDERATION") != "true" {
-		return finishRunning(exec.Command("./hack/ginkgo-e2e.sh", strings.Fields(*testArgs)...))
+		return finishRunning(exec.Command("./hack/ginkgo-e2e.sh", strings.Fields(testArgs)...))
 	}
 
-	if *testArgs == "" {
-		*testArgs = "--ginkgo.focus=\\[Feature:Federation\\]"
+	if testArgs == "" {
+		testArgs = "--ginkgo.focus=\\[Feature:Federation\\]"
 	}
-	return finishRunning(exec.Command("./hack/federated-ginkgo-e2e.sh", strings.Fields(*testArgs)...))
+	return finishRunning(exec.Command("./hack/federated-ginkgo-e2e.sh", strings.Fields(testArgs)...))
 }
diff --git a/kubetest/extract.go b/kubetest/extract.go
new file mode 100644
index 000000000000..0286903ce01b
--- /dev/null
+++ b/kubetest/extract.go
@@ -0,0 +1,368 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+type extractMode int
+
+const (
+	none    extractMode = iota
+	local               // local
+	gci                 // gci/FAMILY
+	gciCi               // gci/FAMILY/CI_VERSION
+	gke                 // gke, gke-staging, gke-test
+	ci                  // ci/latest, ci/latest-1.5
+	rc                  // release/latest, release/latest-1.5
+	stable              // release/stable, release/stable-1.5
+	version             // v1.5.0, v1.5.0-beta.2
+	gcs                 // gs://bucket/prefix/v1.6.0-alpha.0
+)
+
+type extractStrategy struct {
+	mode      extractMode
+	option    string
+	ciVersion string
+	value     string
+}
+
+type extractStrategies []extractStrategy
+
+func (l *extractStrategies) String() string {
+	s := []string{}
+	for _, e := range *l {
+		s = append(s, e.value)
+	}
+	return strings.Join(s, ",")
+}
+
+// Converts --extract=release/stable, etc into an extractStrategy{}
+func (l *extractStrategies) Set(value string) error {
+	var strategies = map[string]extractMode{
+		`^(local)`:                  local,
+		`^gke-?(staging|test)`:      gke,
+		`^gci/([\w-]+)$`:            gci,
+		`^gci/([\w-]+)/(.+)$`:       gciCi,
+		`^ci/(.+)$`:                 ci,
+		`^release/(latest.*)$`:      rc,
+		`^release/(stable.*)$`:      stable,
+		`^(v\d+\.\d+\.\d+[\w.-]*)$`: version,
+		`^(gs://.*)$`:               gcs,
+	}
+
+	if len(*l) == 2 {
+		return fmt.Errorf("May only define at most 2 --extract strategies: %v %v", *l, value)
+	}
+	for search, mode := range strategies {
+		re := regexp.MustCompile(search)
+		mat := re.FindStringSubmatch(value)
+		if mat == nil {
+			continue
+		}
+		e := extractStrategy{
+			mode:   mode,
+			option: mat[1],
+			value:  value,
+		}
+		if len(mat) > 2 {
+			e.ciVersion = mat[2]
+		}
+		*l = append(*l, e)
+		return nil
+	}
+	return fmt.Errorf("Unknown extraction strategy: %v", value)
+
+}
+
+// True when this kubetest invocation wants to download and extract a release.
+func (l *extractStrategies) Enabled() bool {
+	return len(*l) > 0
+}
+
+func (e extractStrategy) name() string {
+	return filepath.Base(e.option)
+}
+
+func (l extractStrategies) Extract() error {
+	// rm -rf kubernetes*
+	if files, err := ioutil.ReadDir("."); err != nil {
+		return err
+	} else {
+		for _, file := range files {
+			name := file.Name()
+			if !strings.HasPrefix(name, "kubernetes") {
+				continue
+			}
+			log.Printf("rm %s", name)
+			if err = os.RemoveAll(name); err != nil {
+				return err
+			}
+		}
+	}
+
+	for i, e := range l {
+		if i > 0 {
+			// TODO(fejta): new strategy so we support more than 2 --extracts
+			if err := os.Rename("kubernetes", "kubernetes_skew"); err != nil {
+				return err
+			}
+		}
+		if err := e.Extract(); err != nil {
+			return err
+		}
+	}
+
+	return os.Chdir("kubernetes")
+}
+
+// Find get-kube.sh at PWD, in PATH or else download it.
+func ensureKube() (string, error) {
+	// Does get-kube.sh exist in pwd?
+	i, err := os.Stat("./get-kube.sh")
+	if err == nil && !i.IsDir() && i.Mode()&0111 > 0 {
+		return "./get-kube.sh", nil
+	}
+
+	// How about in the path?
+	p, err := exec.LookPath("get-kube.sh")
+	if err == nil {
+		return p, nil
+	}
+
+	// Download it to a temp file
+	f, err := ioutil.TempFile("", "get-kube")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	if err := httpRead("https://get.k8s.io", f); err != nil {
+		return "", err
+	}
+	i, err = f.Stat()
+	if err != nil {
+		return "", err
+	}
+	if err := os.Chmod(f.Name(), i.Mode()|0111); err != nil {
+		return "", err
+	}
+	return f.Name(), nil
+}
+
+// Download test binaries for kubernetes versions before 1.5.
+func getTestBinaries(url, version string) error {
+	f, err := os.Create("kubernetes-test.tar.gz")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	full := fmt.Sprintf("%v/%v/kubernetes-test.tar.gz", url, version)
+	if err := httpRead(full, f); err != nil {
+		return err
+	}
+	f.Close()
+	o, err := combinedOutput(exec.Command("md5sum", f.Name()))
+	if err != nil {
+		return err
+	}
+	log.Printf("md5sum: %s", o)
+	if err = finishRunning(exec.Command("tar", "-xzf", f.Name())); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Calls KUBERNETES_RELASE_URL=url KUBERNETES_RELEASE=version get-kube.sh.
+// This will download version from the specified url subdir and extract
+// the tarballs.
+func getKube(url, version string) error {
+	k, err := ensureKube()
+	if err != nil {
+		return err
+	}
+	if err := os.Setenv("KUBERNETES_RELEASE_URL", url); err != nil {
+		return err
+	}
+
+	if err := os.Setenv("KUBERNETES_RELEASE", version); err != nil {
+		return err
+	}
+	if err := os.Setenv("KUBERNETES_SKIP_CONFIRM", "y"); err != nil {
+		return err
+	}
+	if err := os.Setenv("KUBERNETES_SKIP_CREATE_CLUSTER", "y"); err != nil {
+		return err
+	}
+	if err := os.Setenv("KUBERNETES_DOWNLOAD_TESTS", "y"); err != nil {
+		return err
+	}
+	log.Printf("U=%s R=%s get-kube.sh", url, version)
+	if err := finishRunning(exec.Command(k)); err != nil {
+		return err
+	}
+	i, err := os.Stat("./kubernetes/cluster/get-kube-binaries.sh")
+	if err != nil || i.IsDir() {
+		log.Printf("Grabbing test binaries since R=%s < 1.5", version)
+		if err = getTestBinaries(url, version); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func setReleaseFromGcs(ci bool, suffix string) error {
+	var prefix string
+	if ci {
+		prefix = "kubernetes-release-dev/ci"
+	} else {
+		prefix = "kubernetes-release/release"
+	}
+
+	url := fmt.Sprintf("https://storage.googleapis.com/%v", prefix)
+	cat := fmt.Sprintf("gs://%v/%v.txt", prefix, suffix)
+	release, err := combinedOutput(exec.Command("gsutil", "cat", cat))
+	if err != nil {
+		return err
+	}
+	return getKube(url, strings.TrimSpace(string(release)))
+}
+
+func setupGciVars(family string) (string, error) {
+	p := "container-vm-image-staging"
+	b, err := combinedOutput(exec.Command("gcloud", "compute", "images", "describe-from-family", family, fmt.Sprintf("--project=%v", p), "--format=value(name)"))
+	if err != nil {
+		return "", err
+	}
+	i := string(b)
+	g := "gci"
+	m := map[string]string{
+		"KUBE_GCE_MASTER_PROJECT":     p,
+		"KUBE_GCE_MASTER_IMAGE":       i,
+		"KUBE_MASTER_OS_DISTRIBUTION": g,
+
+		"KUBE_GCE_NODE_PROJECT":     p,
+		"KUBE_GCE_NODE_IMAGE":       i,
+		"KUBE_NODE_OS_DISTRIBUTION": g,
+
+		"BUILD_METADATA_GCE_MASTER_IMAGE": i,
+		"BUILD_METADATA_GCE_NODE_IMAGE":   i,
+
+		"KUBE_OS_DISTRIBUTION": g,
+	}
+	if family == "gci-canary-test" {
+		var b bytes.Buffer
+		if err := httpRead("https://api.github.com/repos/docker/docker/releases", &b); err != nil {
+			return "", err
+		}
+		var v []map[string]interface{}
+		if err := json.NewDecoder(&b).Decode(&v); err != nil {
+			return "", err
+		}
+		// We want 1.13.0
+		m["KUBE_GCI_DOCKER_VERSION"] = v[0]["name"].(string)[1:]
+	}
+	for k, v := range m {
+		log.Printf("export %s=%s", k, v)
+		if err := os.Setenv(k, v); err != nil {
+			return "", err
+		}
+	}
+	return i, nil
+}
+
+func setReleaseFromGci(image string) error {
+	u := fmt.Sprintf("gs://container-vm-image-staging/k8s-version-map/%s", image)
+	b, err := combinedOutput(exec.Command("gsutil", "cat", u))
+	if err != nil {
+		return err
+	}
+	r := fmt.Sprintf("v%s", b)
+	return getKube("https://storage.googleapis.com/kubernetes-release/release", strings.TrimSpace(r))
+}
+
+func (e extractStrategy) Extract() error {
+	switch e.mode {
+	case local:
+		url := "./_output/gcs-stage"
+		files, err := ioutil.ReadDir(url)
+		if err != nil {
+			return err
+		}
+		var release string
+		for _, file := range files {
+			r := file.Name()
+			if strings.HasPrefix(r, "v") {
+				release = r
+				break
+			}
+		}
+		if len(release) == 0 {
+			return fmt.Errorf("No releases found in %v", url)
+		}
+		return getKube(url, release)
+	case gci, gciCi:
+		if i, err := setupGciVars(e.option); err != nil {
+			return err
+		} else if e.ciVersion != "" {
+			return setReleaseFromGcs(true, e.ciVersion)
+		} else {
+			return setReleaseFromGci(i)
+		}
+	case gke:
+		// TODO(fejta): prod v staging v test
+		p := os.Getenv("PROJECT")
+		if len(p) == 0 {
+			return fmt.Errorf("PROJECT is unset")
+		}
+		z := os.Getenv("ZONE")
+		if len(z) == 0 {
+			return fmt.Errorf("ZONE is unset")
+		}
+		ci, err := combinedOutput(exec.Command("gcloud", "container", "get-server-config", fmt.Sprintf("--project=%v", p), fmt.Sprintf("--zone=%v", z), "--format=value(defaultClusterVersion)"))
+		if err != nil {
+			return err
+		}
+		return setReleaseFromGcs(true, strings.TrimSpace(string(ci)))
+	case ci:
+		return setReleaseFromGcs(true, e.option)
+	case rc, stable:
+		return setReleaseFromGcs(false, e.option)
+	case version:
+		var url string
+		release := e.option
+		if strings.Contains(release, "+") {
+			url = "https://storage.googleapis.com/kubernetes-release-dev/ci"
+		} else {
+			url = "https://storage.googleapis.com/kubernetes-release/release"
+		}
+		return getKube(url, release)
+	case gcs:
+		return getKube(path.Dir(e.option), path.Base(e.option))
+	}
+	return fmt.Errorf("Unrecognized extraction: %v(%v)", e.mode, e.value)
+}
diff --git a/kubetest/federation.go b/kubetest/federation.go
index df1ed18852ae..778c945878fa 100644
--- a/kubetest/federation.go
+++ b/kubetest/federation.go
@@ -20,12 +20,10 @@ import (
 	"os/exec"
 )
 
-type federationDeployer struct{}
-
-func (f federationDeployer) Up() error {
+func FedUp() error {
 	return finishRunning(exec.Command("./federation/cluster/federation-up.sh"))
 }
 
-func (f federationDeployer) Down() error {
+func FedDown() error {
 	return finishRunning(exec.Command("./federation/cluster/federation-down.sh"))
 }
diff --git a/kubetest/main.go b/kubetest/main.go
index 710bd603c958..6cb664b80e9d 100644
--- a/kubetest/main.go
+++ b/kubetest/main.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2014 The Kubernetes Authors.
+Copyright 2017 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,12 +17,17 @@ limitations under the License.
 package main
 
 import (
+	"encoding/json"
+	"encoding/xml"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 )
@@ -30,34 +35,76 @@ import (
 var (
 	interrupt = time.NewTimer(time.Duration(0)) // interrupt testing at this time.
 	terminate = time.NewTimer(time.Duration(0)) // terminate testing at this time.
-	// TODO(fejta): change all these _ flags to -
-	build            = flag.Bool("build", false, "If true, build a new release. Otherwise, use whatever is there.")
-	checkVersionSkew = flag.Bool("check_version_skew", true, ""+
-		"By default, verify that client and server have exact version match. "+
-		"You can explicitly set to false if you're, e.g., testing client changes "+
-		"for which the server version doesn't make a difference.")
-	checkLeakedResources = flag.Bool("check_leaked_resources", false, "Ensure project ends with the same resources")
-	deployment           = flag.String("deployment", "bash", "up/down mechanism (defaults to cluster/kube-{up,down}.sh) (choices: none/bash/kops/kubernetes-anywhere)")
-	down                 = flag.Bool("down", false, "If true, tear down the cluster before exiting.")
-	dump                 = flag.String("dump", "", "If set, dump cluster logs to this location on test or cluster-up failure")
-	federation           = flag.Bool("federation", false, "If true, start/tear down the federation control plane along with the clusters. To only start/tear down the federation control plane, specify --deploy=none")
-	kubemark             = flag.Bool("kubemark", false, "If true, run kubemark tests.")
-	chartTests           = flag.Bool("charts", false, "If true, run charts tests.")
-	publish              = flag.String("publish", "", "Publish version to the specified gs:// path on success")
-	skewTests            = flag.Bool("skew", false, "If true, run tests in another version at ../kubernetes/hack/e2e.go")
-	testArgs             = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
-	test                 = flag.Bool("test", false, "Run Ginkgo tests.")
-	timeout              = flag.Duration("timeout", time.Duration(0), "Terminate testing after the timeout duration (s/m/h)")
-	up                   = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.")
-	upgradeArgs          = flag.String("upgrade_args", "", "If set, run upgrade tests before other tests")
-	verbose              = flag.Bool("v", false, "If true, print all command output.")
-
-	// Deprecated flags.
-	deprecatedPush   = flag.Bool("push", false, "Deprecated. Does nothing.")
-	deprecatedPushup = flag.Bool("pushup", false, "Deprecated. Does nothing.")
-	deprecatedCtlCmd = flag.String("ctl", "", "Deprecated. Does nothing.")
+	verbose   = false
+	timeout   = time.Duration(0)
 )
 
+type options struct {
+	build       buildStrategy
+	charts      bool
+	checkLeaks  bool
+	checkSkew   bool
+	deployment  string
+	down        bool
+	dump        string
+	extract     extractStrategies
+	federation  bool
+	kubemark    bool
+	publish     string
+	save        string
+	skew        bool
+	stage       stageStrategy
+	testArgs    string
+	test        bool
+	up          bool
+	upgradeArgs string
+}
+
+func defineFlags() (*options, string) {
+	o := options{}
+	var deployment string
+	flag.Var(&o.build, "build", "Rebuild k8s binaries, optionally forcing (make|quick|bazel) stategy")
+	flag.BoolVar(&o.charts, "charts", false, "If true, run charts tests")
+	flag.BoolVar(&o.checkSkew, "check-version-skew", true, "Verify client and server versions match")
+	flag.BoolVar(&o.checkLeaks, "check-leaked-resources", false, "Ensure project ends with the same resources")
+	flag.StringVar(&deployment, "deployment", "bash", "Choices: bash/kops/kubernetes-anywhere")
+	flag.BoolVar(&o.down, "down", false, "If true, tear down the cluster before exiting.")
+	flag.StringVar(&o.dump, "dump", "", "If set, dump cluster logs to this location on test or cluster-up failure")
+	flag.Var(&o.extract, "extract", "Extract k8s binaries from the specified release location")
+	flag.BoolVar(&o.federation, "federation", false, "If true, start/tear down the federation control plane along with the clusters. To only start/tear down the federation control plane, specify --deploy=none")
+	flag.BoolVar(&o.kubemark, "kubemark", false, "If true, run kubemark tests.")
+	flag.StringVar(&o.publish, "publish", "", "Publish version to the specified gs:// path on success")
+	flag.StringVar(&o.save, "save", "", "Save credentials to gs:// path on --up if set (or load from there if not --up)")
+	flag.BoolVar(&o.skew, "skew", false, "If true, run tests in another version at ../kubernetes/hack/e2e.go")
+	flag.Var(&o.stage, "stage", "Upload binaries to gs://bucket/devel/job-suffix if set")
+	flag.BoolVar(&o.test, "test", false, "Run Ginkgo tests.")
+	flag.StringVar(&o.testArgs, "test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
+	flag.DurationVar(&timeout, "timeout", time.Duration(0), "Terminate testing after the timeout duration (s/m/h)")
+	flag.BoolVar(&o.up, "up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.")
+	flag.StringVar(&o.upgradeArgs, "upgrade_args", "", "If set, run upgrade tests before other tests")
+
+	flag.BoolVar(&verbose, "v", false, "If true, print all command output.")
+	return &o, deployment
+}
+
+type testCase struct {
+	XMLName   xml.Name `xml:"testcase"`
+	ClassName string   `xml:"classname,attr"`
+	Name      string   `xml:"name,attr"`
+	Time      float64  `xml:"time,attr"`
+	Failure   string   `xml:"failure,omitempty"`
+}
+
+type TestSuite struct {
+	XMLName  xml.Name `xml:"testsuite"`
+	Failures int      `xml:"failures,attr"`
+	Tests    int      `xml:"tests,attr"`
+	Time     float64  `xml:"time,attr"`
+	Cases    []testCase
+}
+
+var suite TestSuite
+
 func validWorkingDirectory() error {
 	cwd, err := os.Getwd()
 	if err != nil {
@@ -74,9 +121,59 @@ func validWorkingDirectory() error {
 	return nil
 }
 
+func writeXML(dump string, start time.Time) {
+	suite.Time = time.Since(start).Seconds()
+	out, err := xml.MarshalIndent(&suite, "", "    ")
+	if err != nil {
+		log.Fatalf("Could not marshal XML: %s", err)
+	}
+	path := filepath.Join(dump, "junit_runner.xml")
+	f, err := os.Create(path)
+	if err != nil {
+		log.Fatalf("Could not create file: %s", err)
+	}
+	defer f.Close()
+	if _, err := f.WriteString(xml.Header); err != nil {
+		log.Fatalf("Error writing XML header: %s", err)
+	}
+	if _, err := f.Write(out); err != nil {
+		log.Fatalf("Error writing XML data: %s", err)
+	}
+	log.Printf("Saved XML output to %s.", path)
+}
+
+type deployer interface {
+	Up() error
+	IsUp() error
+	SetupKubecfg() error
+	Down() error
+}
+
+func getDeployer(deployment string) (deployer, error) {
+	switch deployment {
+	case "bash":
+		return bash{}, nil
+	case "kops":
+		return NewKops()
+	case "kubernetes-anywhere":
+		return NewKubernetesAnywhere()
+	case "none":
+		return noneDeploy{}, nil
+	default:
+		return nil, fmt.Errorf("Unknown deployment strategy %q", deployment)
+	}
+}
+
 func main() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
+	o, deployment := defineFlags()
 	flag.Parse()
+	if err := complete(o, deployment); err != nil {
+		log.Fatalf("Something went wrong: %s", err)
+	}
+}
+
+func complete(o *options, deployment string) error {
 
 	if !terminate.Stop() {
 		<-terminate.C // Drain the value if necessary.
@@ -85,26 +182,32 @@ func main() {
 		<-interrupt.C // Drain value
 	}
 
-	if *timeout > 0 {
-		log.Printf("Limiting testing to %s", *timeout)
-		interrupt.Reset(*timeout)
+	if timeout > 0 {
+		log.Printf("Limiting testing to %s", timeout)
+		interrupt.Reset(timeout)
 	}
 
-	if err := validWorkingDirectory(); err != nil {
-		log.Fatalf("Called from invalid working directory: %v", err)
+	if o.dump != "" {
+		defer writeMetadata(o.dump)
+		defer writeXML(o.dump, time.Now())
+	}
+	if err := prepare(); err != nil {
+		return err
+	}
+	if err := acquireKubernetes(o); err != nil {
+		return err
 	}
 
-	deploy, err := getDeployer()
-	if err != nil {
-		log.Fatalf("Error creating deployer: %v", err)
+	if err := validWorkingDirectory(); err != nil {
+		return fmt.Errorf("Called from invalid working directory: %v", err)
 	}
 
-	fedDeploy, err := getFederationDeployer()
+	deploy, err := getDeployer(deployment)
 	if err != nil {
-		log.Fatalf("Error creating federation deployer: %v", err)
+		return fmt.Errorf("Error creating deployer: %v", err)
 	}
 
-	if *down {
+	if o.down {
 		// listen for signals such as ^C and gracefully attempt to clean up
 		c := make(chan os.Signal, 1)
 		signal.Notify(c, os.Interrupt)
@@ -112,8 +215,8 @@ func main() {
 			for range c {
 				log.Print("Captured ^C, gracefully attempting to cleanup resources..")
 				var fedErr, err error
-				if *federation {
-					if fedErr = fedDeploy.Down(); fedErr != nil {
+				if o.federation {
+					if fedErr = FedDown(); fedErr != nil {
 						log.Printf("Tearing down federation failed: %v", fedErr)
 					}
 				}
@@ -127,7 +230,256 @@ func main() {
 		}()
 	}
 
-	if err := run(deploy, fedDeploy); err != nil {
-		log.Fatalf("Something went wrong: %s", err)
+	if err := run(deploy, *o); err != nil {
+		return err
+	}
+
+	if err := postpare(o.publish, o.save, os.Getenv("KUBERNETES_RELEASE_URL"), os.Getenv("KUBERNETES_RELEASE")); err != nil {
+		return err
+	}
+	return nil
+}
+
+func acquireKubernetes(o *options) error {
+	// Potentially build kubernetes
+	if o.build.Enabled() {
+		if err := xmlWrap("Build", o.build.Build); err != nil {
+			return err
+		}
+	}
+
+	// Potentailly stage build binaries somewhere on GCS
+	if o.stage.Enabled() {
+		if err := xmlWrap("Stage", o.stage.Stage); err != nil {
+			return err
+		}
+	}
+
+	// Potentially download existing binaries and extract them.
+	if o.extract.Enabled() {
+		if err := xmlWrap("Extract", o.extract.Extract); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Returns the k8s version name
+func findVersion() string {
+	// The version may be in a version file
+	if _, err := os.Stat("version"); err == nil {
+		if b, err := ioutil.ReadFile("version"); err == nil {
+			return string(b)
+		} else {
+			log.Printf("Failed to read version: %v", err)
+		}
+	}
+
+	// We can also get it from the git repo.
+	if _, err := os.Stat("hack/lib/version.sh"); err == nil {
+		// TODO(fejta): do this in go. At least we removed the upload-to-gcs.sh dep.
+		gross := `. hack/lib/version.sh && KUBE_ROOT=. kube::version::get_version_vars && echo "${KUBE_GIT_VERSION-}"`
+		if b, err := combinedOutput(exec.Command("bash", "-c", gross)); err == nil {
+			return string(b)
+		} else {
+			log.Printf("Failed to get_version_vars: %v", err)
+		}
+	}
+
+	return "unknown" // Sad trombone
+}
+
+// Write metadata.json, including version and env arg data.
+func writeMetadata(path string) error {
+	m := make(map[string]string)
+	ver := findVersion()
+	m["version"] = ver // TODO(fejta): retire
+	m["job-version"] = ver
+	re := regexp.MustCompile(`^BUILD_METADATA_(.+)$`)
+	for _, e := range os.Environ() {
+		p := strings.SplitN(e, "=", 2)
+		r := re.FindStringSubmatch(p[0])
+		if r == nil {
+			continue
+		}
+		k, v := strings.ToLower(r[1]), p[1]
+		m[k] = v
+	}
+	f, err := os.Create(filepath.Join(path, "metadata.json"))
+	if err != nil {
+		return err
 	}
+	defer f.Close()
+	e := json.NewEncoder(f)
+	return e.Encode(m)
+}
+
+// Install cloudsdk tarball to location, updating PATH
+func installGcloud(tarball string, location string) error {
+
+	if err := finishRunning(exec.Command("tar", "xzf", tarball, "-C", location)); err != nil {
+		return err
+	}
+
+	if err := finishRunning(exec.Command(filepath.Join(location, "google-cloud-sdk", "install.sh"), "--disable-installation-options", "--bash-completion=false", "--path-update=false", "--usage-reporting=false")); err != nil {
+		return err
+	}
+
+	if err := insertPath(filepath.Join(location, "google-cloud-sdk", "bin")); err != nil {
+		return err
+	}
+
+	if err := finishRunning(exec.Command("gcloud", "components", "install", "alpha")); err != nil {
+		return err
+	}
+
+	if err := finishRunning(exec.Command("gcloud", "components", "install", "beta")); err != nil {
+		return err
+	}
+
+	if err := finishRunning(exec.Command("gcloud", "info")); err != nil {
+		return err
+	}
+	return nil
+}
+
+func prepareGcp(kubernetesProvider string) error {
+	// Ensure project is set
+	p := os.Getenv("PROJECT")
+	if p == "" {
+		return fmt.Errorf("KUBERNETES_PROVIDER=%s requires setting PROJECT", kubernetesProvider)
+	}
+
+	// gcloud creds may have changed
+	if err := activateServiceAccount(); err != nil {
+		return err
+	}
+
+	// Ensure ssh keys exist
+	log.Print("Checking existing of GCP ssh keys...")
+	k := filepath.Join(".ssh", "google_compute_engine")
+	if _, err := os.Stat(k); err != nil {
+		return err
+	}
+	pk := k + ".pub"
+	if _, err := os.Stat(pk); err != nil {
+		return err
+	}
+
+	log.Printf("Checking presence of public key in %s", p)
+	if o, err := combinedOutput(exec.Command("gcloud", "compute", "--project="+p, "project-info", "describe")); err != nil {
+		return err
+	} else if b, err := ioutil.ReadFile(pk); err != nil {
+		return err
+	} else if !strings.Contains(string(b), string(o)) {
+		log.Print("Uploading public ssh key to project metadata...")
+		if err = finishRunning(exec.Command("gcloud", "compute", "--project="+p, "config-ssh")); err != nil {
+			return err
+		}
+	}
+
+	// Install custom gcloud verion if necessary
+	if b := os.Getenv("CLOUDSDK_BUCKET"); b != "" {
+
+		for i := 0; i < 3; i++ {
+			if err := finishRunning(exec.Command("gsutil", "-mq", "cp", "-r", b, home())); err == nil {
+				break // Success!
+			}
+			time.Sleep(1 << uint(i) * time.Second)
+		}
+		for _, f := range []string{home(".gsutil"), home("repo"), home("cloudsdk")} {
+			if _, err := os.Stat(f); err == nil || !os.IsNotExist(err) {
+				if err = os.RemoveAll(f); err != nil {
+					return err
+				}
+			}
+		}
+		if err := os.Rename(home(filepath.Base(b)), home("repo")); err != nil {
+			return err
+		}
+		// Controls which gcloud components to install.
+		ccmsu := "CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL"
+		defer os.Setenv(ccmsu, os.Getenv(ccmsu))
+		os.Setenv(ccmsu, "file://"+home("repo", "components-2.json"))
+		if err := installGcloud(home("repo", "google-cloud-sdk.tar.gz"), home("cloudsdk")); err != nil {
+			return err
+		}
+		// gcloud creds may have changed
+		if err := activateServiceAccount(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func prepareAws() error {
+	return finishRunning(exec.Command("pip", "install", "awscli"))
+}
+
+// Activate GOOGLE_APPLICATION_CREDENTIALS if set or do nothing.
+func activateServiceAccount() error {
+	if k := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"); k != "" {
+		return finishRunning(exec.Command("gcloud", "auth", "activate-service-account", "--key-file="+k))
+	}
+	return nil
+}
+
+func prepare() error {
+	kp := os.Getenv("KUBERNETES_PROVIDER")
+	switch kp {
+	case "gce", "gke", "kubemark":
+		if err := prepareGcp(kp); err != nil {
+			return err
+		}
+	case "aws":
+		if err := prepareAws(); err != nil {
+			return err
+		}
+	}
+
+	if err := activateServiceAccount(); err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll("./_artifacts", 0777); err != nil { // Create artifacts
+		return err
+	}
+
+	if p := os.Getenv("PRIORITY_PATH"); p != "" {
+		if err := insertPath(p); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func postpare(publish, save, url, version string) error {
+	if publish != "" {
+		if v, err := ioutil.ReadFile("version"); err != nil {
+			return err
+		} else {
+			log.Printf("Set %s version to %s", publish, string(v))
+		}
+		if err := finishRunning(exec.Command("gsutil", "cp", "version", publish)); err != nil {
+			return err
+		}
+	}
+	if save != "" {
+		if err := finishRunning(exec.Command("gsutil", "cp", home(".kube", "config"), filepath.Join(save, "kube-config"))); err != nil {
+			return err
+		}
+		if cmd, err := inputCommand(url, "gsutil", "cp", "-", filepath.Join(save, "release-url.txt")); err != nil {
+			return err
+		} else if err = finishRunning(cmd); err != nil {
+			return err
+		}
+
+		if cmd, err := inputCommand(version, "gsutil", "cp", "-", filepath.Join(save, "release.txt")); err != nil {
+			return err
+		} else if err = finishRunning(cmd); err != nil {
+			return err
+		}
+	}
+	return nil
+	// chmod -R o+r everything
 }
diff --git a/kubetest/none.go b/kubetest/none.go
index 90ae092fbe82..23c234c92b88 100644
--- a/kubetest/none.go
+++ b/kubetest/none.go
@@ -16,20 +16,28 @@ limitations under the License.
 
 package main
 
-type none struct{}
+import (
+	"log"
+)
 
-func (n none) Up() error {
+type noneDeploy struct{}
+
+func (n noneDeploy) Up() error {
+	log.Print("Noop Up()")
 	return nil
 }
 
-func (n none) IsUp() error {
+func (n noneDeploy) IsUp() error {
+	log.Print("Noop IsUp()")
 	return nil
 }
 
-func (n none) SetupKubecfg() error {
+func (n noneDeploy) SetupKubecfg() error {
+	log.Print("Noop SetupKubecfg()")
 	return nil
 }
 
-func (n none) Down() error {
+func (n noneDeploy) Down() error {
+	log.Print("Noop Down()")
 	return nil
 }
diff --git a/kubetest/stage.go b/kubetest/stage.go
new file mode 100644
index 000000000000..9dc0cdd822f3
--- /dev/null
+++ b/kubetest/stage.go
@@ -0,0 +1,79 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"regexp"
+)
+
+type stageStrategy struct {
+	bucket string
+	ci     bool
+	suffix string
+}
+
+// Return something like gs://bucket/ci/suffix
+func (s *stageStrategy) String() string {
+	p := "devel"
+	if s.ci {
+		p = "ci"
+	}
+	return fmt.Sprintf("%v%v%v", s.bucket, p, s.suffix)
+}
+
+// Parse bucket, ci, suffix from gs://BUCKET/ci/SUFFIX
+func (s *stageStrategy) Set(value string) error {
+	re := regexp.MustCompile(`^(gs://[\w-]+)/(devel|ci)(/.*)?`)
+	mat := re.FindStringSubmatch(value)
+	if mat == nil {
+		return fmt.Errorf("Invalid stage location: %v. Use gs://bucket/ci/optional-suffix", value)
+	}
+	s.bucket = mat[1]
+	s.ci = mat[2] == "ci"
+	s.suffix = mat[3]
+	return nil
+}
+
+// True when this kubetest invocation wants to stage the release
+func (s *stageStrategy) Enabled() bool {
+	return s.bucket != ""
+}
+
+// Stage the release build to GCS.
+// Essentially release/push-build.sh --bucket=B --ci? --gcs-suffix=S --federation?
+func (s *stageStrategy) Stage() error {
+	name := "../release/push-build.sh"
+	args := []string{
+		"--nomock",
+		"--verbose",
+		fmt.Sprintf("--bucket=%v", s.bucket),
+	}
+	if s.ci {
+		args = append(args, "--ci")
+	}
+	if len(s.suffix) > 0 {
+		args = append(args, fmt.Sprintf("--gcs-suffix=%v", s.suffix))
+	}
+	if os.Getenv("FEDERATION") == "true" {
+		args = append(args, "--federation")
+	}
+
+	return finishRunning(exec.Command(name, args...))
+}
diff --git a/kubetest/util.go b/kubetest/util.go
index 584dd43f329c..fa7602adcc8f 100644
--- a/kubetest/util.go
+++ b/kubetest/util.go
@@ -17,9 +17,10 @@ limitations under the License.
 package main
 
 import (
-	"encoding/xml"
 	"fmt"
+	"io"
 	"log"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -28,43 +29,44 @@ import (
 	"time"
 )
 
-type testCase struct {
-	XMLName   xml.Name `xml:"testcase"`
-	ClassName string   `xml:"classname,attr"`
-	Name      string   `xml:"name,attr"`
-	Time      float64  `xml:"time,attr"`
-	Failure   string   `xml:"failure,omitempty"`
+// append(errs, err) if err != nil
+func appendError(errs []error, err error) []error {
+	if err != nil {
+		return append(errs, err)
+	}
+	return errs
 }
 
-type testSuite struct {
-	XMLName  xml.Name `xml:"testsuite"`
-	Failures int      `xml:"failures,attr"`
-	Tests    int      `xml:"tests,attr"`
-	Time     float64  `xml:"time,attr"`
-	Cases    []testCase
+// Returns $HOME/part/part/part
+func home(parts ...string) string {
+	p := []string{os.Getenv("HOME")}
+	for _, a := range parts {
+		p = append(p, a)
+	}
+	return filepath.Join(p...)
 }
 
-var suite testSuite
+// export PATH=path:$PATH
+func insertPath(path string) error {
+	return os.Setenv("PATH", fmt.Sprintf("%v:%v", path, os.Getenv("PATH")))
+}
 
-func writeXML(start time.Time) {
-	suite.Time = time.Since(start).Seconds()
-	out, err := xml.MarshalIndent(&suite, "", "    ")
-	if err != nil {
-		log.Fatalf("Could not marshal XML: %s", err)
-	}
-	path := filepath.Join(*dump, "junit_runner.xml")
-	f, err := os.Create(path)
+// Essentially curl url | writer
+func httpRead(url string, writer io.Writer) error {
+	log.Printf("curl %s", url)
+	r, err := http.Get(url)
 	if err != nil {
-		log.Fatalf("Could not create file: %s", err)
+		return err
 	}
-	defer f.Close()
-	if _, err := f.WriteString(xml.Header); err != nil {
-		log.Fatalf("Error writing XML header: %s", err)
+	defer r.Body.Close()
+	if r.StatusCode >= 400 {
+		return fmt.Errorf("%v returned %d", url, r.StatusCode)
 	}
-	if _, err := f.Write(out); err != nil {
-		log.Fatalf("Error writing XML data: %s", err)
+	_, err = io.Copy(writer, r.Body)
+	if err != nil {
+		return err
 	}
-	log.Printf("Saved XML output to %s.", path)
+	return nil
 }
 
 // return f(), adding junit xml testcase result for name
@@ -89,7 +91,7 @@ func xmlWrap(name string, f func() error) error {
 // return cmd.Wait() and/or timing out.
 func finishRunning(cmd *exec.Cmd) error {
 	stepName := strings.Join(cmd.Args, " ")
-	if *verbose {
+	if verbose {
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
 	}
@@ -129,3 +131,60 @@ func finishRunning(cmd *exec.Cmd) error {
 		}
 	}
 }
+
+// return exec.Command(cmd, args...) while calling .StdinPipe().WriteString(input)
+func inputCommand(input, cmd string, args ...string) (*exec.Cmd, error) {
+	c := exec.Command(cmd, args...)
+	w, e := c.StdinPipe()
+	if e != nil {
+		return nil, e
+	}
+	go func() {
+		if _, e = io.WriteString(w, input); e != nil {
+			log.Printf("Failed to write all %d chars to %s: %v", len(input), cmd, e)
+		}
+		if e = w.Close(); e != nil {
+			log.Printf("Failed to close stdin for %s: %v", cmd, e)
+		}
+	}()
+	return c, nil
+}
+
+// return cmd.CombinedOutput(), potentially timing out in the process.
+func combinedOutput(cmd *exec.Cmd) ([]byte, error) {
+	stepName := strings.Join(cmd.Args, " ")
+	log.Printf("Running: %v", stepName)
+	defer func(start time.Time) {
+		log.Printf("Step '%s' finished in %s", stepName, time.Since(start))
+	}(time.Now())
+
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	type result struct {
+		bytes []byte
+		err   error
+	}
+	finished := make(chan result)
+	go func() {
+		b, err := cmd.CombinedOutput()
+		finished <- result{b, err}
+	}()
+	for {
+		select {
+		case <-terminate.C:
+			terminate.Reset(time.Duration(0)) // Kill subsequent processes immediately.
+			syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
+			cmd.Process.Kill()
+			return nil, fmt.Errorf("Terminate testing after 15m after %s timeout during %s", timeout, stepName)
+		case <-interrupt.C:
+			log.Printf("Interrupt testing after %s timeout. Will terminate in another 15m", timeout)
+			terminate.Reset(15 * time.Minute)
+			if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGINT); err != nil {
+				log.Printf("Failed to interrupt %v. Will terminate immediately: %v", stepName, err)
+				syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM)
+				cmd.Process.Kill()
+			}
+		case fin := <-finished:
+			return fin.bytes, fin.err
+		}
+	}
+}