From 27f1fccec41e700cf5433e795c3576d5df06781d Mon Sep 17 00:00:00 2001 From: Andrea Fasano Date: Wed, 20 Oct 2021 11:26:04 +0200 Subject: [PATCH] Using proxy to run the test remotely from the CI pod --- .../baremetalds-devscripts-gather-ref.yaml | 2 +- .../e2e/test/baremetalds-e2e-test-commands.sh | 78 +++++++++++-------- .../e2e/test/baremetalds-e2e-test-ref.yaml | 10 +++ 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/ci-operator/step-registry/baremetalds/devscripts/gather/baremetalds-devscripts-gather-ref.yaml b/ci-operator/step-registry/baremetalds/devscripts/gather/baremetalds-devscripts-gather-ref.yaml index 4c5b7e5626bd5..be56149303272 100644 --- a/ci-operator/step-registry/baremetalds/devscripts/gather/baremetalds-devscripts-gather-ref.yaml +++ b/ci-operator/step-registry/baremetalds/devscripts/gather/baremetalds-devscripts-gather-ref.yaml @@ -6,6 +6,6 @@ ref: resources: requests: cpu: 1000m - memory: 500Mi + memory: 1000Mi documentation: |- Gather additional logs specific to baremetal IPI deployments. diff --git a/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-commands.sh b/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-commands.sh index ac03489bebf45..276e37d188df3 100644 --- a/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-commands.sh +++ b/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-commands.sh @@ -4,29 +4,16 @@ set -o nounset set -o errexit set -o pipefail -echo "************ baremetalds test command ************" - -collect_artifacts() { - echo "$(date +%s)" > "${SHARED_DIR}/TEST_TIME_TEST_END" - - echo "### Fetching results" - ssh "${SSHOPTS[@]}" "root@${IP}" tar -czf - /tmp/artifacts | tar -C "${ARTIFACT_DIR}" -xzf - -} -trap collect_artifacts EXIT TERM - -function copy_test_binaries() { - # Copy test binaries on packet server - echo "### Copying test binaries" - scp "${SSHOPTS[@]}" /usr/bin/openshift-tests /usr/bin/kubectl "root@${IP}:/usr/local/bin" -} - function mirror_test_images() { echo "### Mirroring test images" DEVSCRIPTS_TEST_IMAGE_REPO=${DS_REGISTRY}/localimages/local-test-image + + openshift-tests images --to-repository ${DEVSCRIPTS_TEST_IMAGE_REPO} > /tmp/mirror + scp "${SSHOPTS[@]}" /tmp/mirror "root@${IP}:/tmp/mirror" + # shellcheck disable=SC2087 ssh "${SSHOPTS[@]}" "root@${IP}" bash - << EOF -openshift-tests images --to-repository ${DEVSCRIPTS_TEST_IMAGE_REPO} > /tmp/mirror oc image mirror -f /tmp/mirror --registry-config ${DS_WORKING_DIR}/pull_secret.json EOF TEST_ARGS="--from-repository ${DEVSCRIPTS_TEST_IMAGE_REPO}" @@ -46,29 +33,55 @@ function set_test_provider() { # access. if [[ "${DS_IP_STACK}" != "v6" ]]; then - export TEST_PROVIDER='\{\"type\":\"baremetal\"\}' + export TEST_PROVIDER='{"type":"baremetal"}' else - export TEST_PROVIDER='\{\"type\":\"baremetal\",\"disconnected\":true\}' + export TEST_PROVIDER='{"type":"baremetal","disconnected":true}' fi } +function setup_proxy() { + # For disconnected or otherwise unreachable environments, we want to + # have steps use an HTTP(S) proxy to reach the API server. This proxy + # configuration file should export HTTP_PROXY, HTTPS_PROXY, and NO_PROXY + # environment variables, as well as their lowercase equivalents (note + # that libcurl doesn't recognize the uppercase variables). + if test -f "${SHARED_DIR}/proxy-conf.sh" + then + # shellcheck source=/dev/null + source "${SHARED_DIR}/proxy-conf.sh" + fi +} + +function is_openshift_version_gte() { + printf '%s\n%s' "$1" "${DS_OPENSHIFT_VERSION}" | sort -C -V +} + case "${CLUSTER_TYPE}" in packet) # shellcheck source=/dev/null source "${SHARED_DIR}/packet-conf.sh" # shellcheck source=/dev/null source "${SHARED_DIR}/ds-vars.conf" - copy_test_binaries + + setup_proxy + export KUBECONFIG=${SHARED_DIR}/kubeconfig echo "### Checking release version" - if printf '%s\n%s' "4.8" "${DS_OPENSHIFT_VERSION}" | sort -C -V; then + if is_openshift_version_gte "4.8"; then # Set test provider for only versions greater than or equal to 4.8 set_test_provider # Mirroring test images is supported only for versions greater than or equal to 4.8 mirror_test_images + + # Skipping proxy related tests ([Skipped:Proxy]) is supported only for version greater than or equal to 4.10 + # For lower versions they must be skipped manually + if ! is_openshift_version_gte "4.10"; then + TEST_SKIPS="${TEST_SKIPS} +${TEST_SKIPS_PROXY}" + fi else - export TEST_PROVIDER='\{\"type\":\"\skeleton\"\}' + export TEST_PROVIDER='{"type":"skeleton"}' use_minimal_test_list fi ;; @@ -77,36 +90,33 @@ esac function upgrade() { set -x - ssh "${SSHOPTS[@]}" "root@${IP}" \ - openshift-tests run-upgrade all \ + openshift-tests run-upgrade all \ --to-image "${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}" \ --provider "${TEST_PROVIDER:-}" \ - -o "/tmp/artifacts/e2e.log" \ - --junit-dir "/tmp/artifacts/junit" + -o "${ARTIFACT_DIR}/e2e.log" \ + --junit-dir "${ARTIFACT_DIR}/junit" set +x } function suite() { - if [[ -n "${TEST_SKIPS}" ]]; then - TESTS="$(ssh "${SSHOPTS[@]}" "root@${IP}" openshift-tests run --dry-run --provider "${TEST_PROVIDER}" "${TEST_SUITE}")" && + if [[ -n "${TEST_SKIPS}" ]]; then + TESTS="$(openshift-tests run --dry-run --provider "${TEST_PROVIDER}" "${TEST_SUITE}")" && echo "${TESTS}" | grep -v "${TEST_SKIPS}" >/tmp/tests && echo "Skipping tests:" && echo "${TESTS}" | grep "${TEST_SKIPS}" || { exit_code=$?; echo 'Error: no tests were found matching the TEST_SKIPS regex:'; echo "$TEST_SKIPS"; return $exit_code; } && TEST_ARGS="${TEST_ARGS:-} --file /tmp/tests" fi - scp "${SSHOPTS[@]}" /tmp/tests "root@${IP}:/tmp/tests" - set -x - ssh "${SSHOPTS[@]}" "root@${IP}" \ - openshift-tests run "${TEST_SUITE}" "${TEST_ARGS:-}" \ + openshift-tests run "${TEST_SUITE}" ${TEST_ARGS:-} \ --provider "${TEST_PROVIDER:-}" \ - -o "/tmp/artifacts/e2e.log" \ - --junit-dir "/tmp/artifacts/junit" + -o "${ARTIFACT_DIR}/e2e.log" \ + --junit-dir "${ARTIFACT_DIR}/junit" set +x } echo "$(date +%s)" > "${SHARED_DIR}/TEST_TIME_TEST_START" +trap 'echo "$(date +%s)" > "${SHARED_DIR}/TEST_TIME_TEST_END"' EXIT case "${TEST_TYPE}" in upgrade-conformance) diff --git a/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-ref.yaml b/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-ref.yaml index f81cb1ea6efab..4974655fd1d04 100644 --- a/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-ref.yaml +++ b/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-ref.yaml @@ -53,7 +53,17 @@ ref: \[sig-network\] DNS should resolve DNS of partial qualified names for services \[sig-network\] DNS should resolve DNS of partial qualified names for the cluster \[sig-storage\] In-tree Volumes \[Driver: nfs\] \[Testpattern: Dynamic PV (default fs)\] provisioning should provision storage with mount options + \[sig-network-edge\]\[Conformance\]\[Area:Networking\]\[Feature:Router\] The HAProxy router should be able to connect to a service that is idled because a GET on the route will unidle it documentation: Regular expression (POSIX basic regular expression) of tests to skip. Note that the current list it's just a temporary measure for the baremetal platform. + - name: TEST_SKIPS_PROXY + default: |- + \[sig-cli\] Kubectl client Simple pod should support exec through an HTTP proxy + \[sig-cli\] Kubectl client Simple pod should support exec through kubectl proxy + \[sig-node\] Pods should support retrieving logs from the container over websockets + \[sig-cli\] Kubectl Port forwarding With a server listening on localhost should support forwarding over websockets + \[sig-cli\] Kubectl Port forwarding With a server listening on 0.0.0.0 should support forwarding over websockets + \[sig-node\] Pods should support remote command execution over websockets + documentation: Regular expression (POSIX basic regular expression) of tests to skip belonging to the proxied rule - name: TEST_UPGRADE_OPTIONS default: "abort-at=100" documentation: Options controlling how an upgrade is performed. See `openshift-test run-upgrade --help` for more details.