Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

prow.sh: generic driver installation #69

Merged
merged 1 commit into from
Feb 10, 2020
Merged

prow.sh: generic driver installation #69

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 43 additions & 61 deletions prow.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,19 +157,21 @@ csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr
# the caller.
configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"

# The hostpath deployment script is searched for in several places.
# By default, this script tests sidecars with the CSI hostpath driver,
# using the install_csi_driver function. That function depends on
# a deployment script that it searches for in several places:
#
# - The "deploy" directory in the current repository: this is useful
# for the situation that a component becomes incompatible with the
# shared deployment, because then it can (temporarily!) provide its
# own example until the shared one can be updated; it's also how
# csi-driver-host-path itself provides the example.
#
# - CSI_PROW_HOSTPATH_VERSION of the CSI_PROW_HOSTPATH_REPO is checked
# - CSI_PROW_DRIVER_VERSION of the CSI_PROW_DRIVER_REPO is checked
# out: this allows other repos to reference a version of the example
# that is known to be compatible.
#
# - The csi-driver-host-path/deploy directory has multiple sub-directories,
# - The <driver repo>/deploy directory can have multiple sub-directories,
# each with different deployments (stable set of images for Kubernetes 1.13,
# stable set of images for Kubernetes 1.14, canary for latest Kubernetes, etc.).
# This is necessary because there may be incompatible changes in the
Expand All @@ -186,16 +188,26 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
# "none" disables the deployment of the hostpath driver.
#
# When no deploy script is found (nothing in `deploy` directory,
# CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed.
configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc3" "hostpath driver"
configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo"
# CSI_PROW_DRIVER_REPO=none), nothing gets deployed.
#
# If the deployment script is called with CSI_PROW_TEST_DRIVER=<file name> as
# environment variable, then it must write a suitable test driver configuration
# into that file in addition to installing the driver.
configvar CSI_PROW_DRIVER_VERSION "v1.3.0-rc4" "CSI driver version"
configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo"
configvar CSI_PROW_DEPLOYMENT "" "deployment"
configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name"

# If CSI_PROW_HOSTPATH_CANARY is set (typically to "canary", but also
# "1.0-canary"), then all image versions are replaced with that
# version tag.
configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image"
# The install_csi_driver function may work also for other CSI drivers,
# as long as they follow the conventions of the CSI hostpath driver.
# If they don't, then a different install function can be provided in
# a .prow.sh file and this config variable can be overridden.
configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell function which installs the CSI driver"

# If CSI_PROW_DRIVER_CANARY is set (typically to "canary", but also
# version tag. Usually empty. CSI_PROW_HOSTPATH_CANARY is
# accepted as alternative name because some test-infra jobs
# still use that name.
configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images"

# The E2E testing can come from an arbitrary repo. The expectation is that
# the repo supports "go test ./test/e2e -args --storage.testdriver" (https://github.com/kubernetes/kubernetes/pull/72836)
Expand Down Expand Up @@ -613,7 +625,7 @@ find_deployment () {

# Fixed deployment name? Use it if it exists, otherwise fail.
if [ "${CSI_PROW_DEPLOYMENT}" ]; then
file="$dir/${CSI_PROW_DEPLOYMENT}/deploy-hostpath.sh"
file="$dir/${CSI_PROW_DEPLOYMENT}/deploy.sh"
if ! [ -e "$file" ]; then
return 1
fi
Expand All @@ -623,22 +635,21 @@ find_deployment () {

# Ignore: See if you can use ${variable//search/replace} instead.
# shellcheck disable=SC2001
file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy-hostpath.sh"
file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh"
if ! [ -e "$file" ]; then
file="$dir/kubernetes-latest/deploy-hostpath.sh"
file="$dir/kubernetes-latest/deploy.sh"
if ! [ -e "$file" ]; then
return 1
fi
fi
echo "$file"
}

# This installs the hostpath driver example. CSI_PROW_HOSTPATH_CANARY overrides all
# image versions with that canary version. The parameters of install_hostpath can be
# used to override registry and/or tag of individual images (CSI_PROVISIONER_REGISTRY=localhost:9000
# CSI_PROVISIONER_TAG=latest).
install_hostpath () {
local images deploy_hostpath
# This installs the CSI driver. It's called with a list of env variables
# that override the default images. CSI_PROW_DRIVER_CANARY overrides all
# image versions with that canary version.
install_csi_driver () {
local images deploy_driver
images="$*"

if [ "${CSI_PROW_DEPLOYMENT}" = "none" ]; then
Expand All @@ -654,31 +665,31 @@ install_hostpath () {
done
fi

if deploy_hostpath="$(find_deployment "$(pwd)/deploy")"; then
if deploy_driver="$(find_deployment "$(pwd)/deploy")"; then
:
elif [ "${CSI_PROW_HOSTPATH_REPO}" = "none" ]; then
elif [ "${CSI_PROW_DRIVER_REPO}" = "none" ]; then
return 1
else
git_checkout "${CSI_PROW_HOSTPATH_REPO}" "${CSI_PROW_WORK}/hostpath" "${CSI_PROW_HOSTPATH_VERSION}" --depth=1 || die "checking out hostpath repo failed"
if deploy_hostpath="$(find_deployment "${CSI_PROW_WORK}/hostpath/deploy")"; then
git_checkout "${CSI_PROW_DRIVER_REPO}" "${CSI_PROW_WORK}/csi-driver" "${CSI_PROW_DRIVER_VERSION}" --depth=1 || die "checking out CSI driver repo failed"
if deploy_driver="$(find_deployment "${CSI_PROW_WORK}/csi-driver/deploy")"; then
:
else
die "deploy-hostpath.sh not found in ${CSI_PROW_HOSTPATH_REPO} ${CSI_PROW_HOSTPATH_VERSION}. To disable E2E testing, set CSI_PROW_HOSTPATH_REPO=none"
die "deploy.sh not found in ${CSI_PROW_DRIVER_REPO} ${CSI_PROW_DRIVER_VERSION}. To disable E2E testing, set CSI_PROW_DRIVER_REPO=none"
fi
fi

if [ "${CSI_PROW_HOSTPATH_CANARY}" != "stable" ]; then
images="$images IMAGE_TAG=${CSI_PROW_HOSTPATH_CANARY}"
if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then
images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}"
fi
# Ignore: Double quote to prevent globbing and word splitting.
# It's intentional here for $images.
# shellcheck disable=SC2086
if ! run env $images "${deploy_hostpath}"; then
if ! run env "CSI_PROW_TEST_DRIVER=${CSI_PROW_WORK}/test-driver.yaml" $images "${deploy_driver}"; then
# Collect information about failed deployment before failing.
collect_cluster_info
(start_loggers >/dev/null; wait)
info "For container output see job artifacts."
die "deploying the hostpath driver with ${deploy_hostpath} failed"
die "deploying the CSI driver with ${deploy_driver} failed"
fi
}

Expand Down Expand Up @@ -804,33 +815,6 @@ install_sanity () (
run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed"
)

# The default implementation of this function generates a external
# driver test configuration for the hostpath driver.
#
# The content depends on both what the E2E suite expects and what the
# installed hostpath driver supports. Generating it here seems prone
# to breakage, but it is uncertain where a better place might be.
generate_test_driver () {
cat <<EOF
ShortName: csiprow
StorageClass:
FromName: true
SnapshotClass:
FromName: true
DriverInfo:
Name: ${CSI_PROW_HOSTPATH_DRIVER_NAME}
Capabilities:
block: true
persistence: true
dataSource: true
multipods: true
nodeExpansion: true
controllerExpansion: true
snapshotDataSource: true
singleNodeVolume: true
EOF
}

# Captures pod output while running some other command.
run_with_loggers () (
loggers=$(start_loggers -f)
Expand All @@ -852,8 +836,6 @@ run_e2e () (
install_e2e || die "building e2e.test failed"
install_ginkgo || die "installing ginkgo failed"

generate_test_driver >"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed"

# Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again
# and to avoid the large number of "skipped" tests that we get from using
# the full Kubernetes E2E testsuite while only running a few tests.
Expand Down Expand Up @@ -1063,7 +1045,7 @@ main () {
cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')"
# Get the image that was just built (if any) from the
# top-level Makefile CMDS variable and set the
# deploy-hostpath.sh env variables for it. We also need to
# deploy.sh env variables for it. We also need to
# side-load those images into the cluster.
for i in $cmds; do
e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _)
Expand Down Expand Up @@ -1101,7 +1083,7 @@ main () {
fi

# Installing the driver might be disabled.
if install_hostpath "$images"; then
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
collect_cluster_info

if sanity_enabled; then
Expand Down Expand Up @@ -1158,7 +1140,7 @@ main () {
fi

# Installing the driver might be disabled.
if install_hostpath "$images"; then
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
collect_cluster_info

if tests_enabled "parallel-alpha"; then
Expand Down