Refactor baremetalds-e2e-test to be more similar to openshift-e2e-test

ci-operator/step-registry/baremetalds/devscripts/setup/baremetalds-devscripts-setup-commands.sh

@@ -91,5 +91,17 @@ fi
 echo 'export KUBECONFIG=/root/dev-scripts/ocp/ostest/auth/kubeconfig' >> /root/.bashrc
 
 timeout -s 9 105m make
+EOF
+
+# Copy dev-scripts variables to be shared with the test step
+ssh "${SSHOPTS[@]}" "root@${IP}" bash - << EOF |& sed -e 's/.*auths.*/*** PULL_SECRET ***/g'
+cd /root/dev-scripts
+source common.sh
+source ocp_install_env.sh
 
+echo "export DS_OPENSHIFT_VERSION=\$(openshift_version)" >> /tmp/ds-vars.conf
+echo "export DS_REGISTRY=\$LOCAL_REGISTRY_DNS_NAME:\$LOCAL_REGISTRY_PORT" >> /tmp/ds-vars.conf
+echo "export DS_WORKING_DIR=\$WORKING_DIR" >> /tmp/ds-vars.conf
 EOF
+
+scp "${SSHOPTS[@]}" "root@${IP}:/tmp/ds-vars.conf" "${SHARED_DIR}/ds-vars.conf"

ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-commands.sh

@@ -6,101 +6,110 @@ set -o pipefail
 
 echo "************ baremetalds test command ************"
 
-# Fetch packet basic configuration
-# shellcheck source=/dev/null
-source "${SHARED_DIR}/packet-conf.sh"
-
 collect_artifacts() {
+    echo "$(date +%s)" > "${SHARED_DIR}/TEST_TIME_TEST_END"
+
     echo "### Fetching results"
     ssh "${SSHOPTS[@]}" "root@${IP}" tar -czf - /tmp/artifacts | tar -C "${ARTIFACT_DIR}" -xzf -
 }
 trap collect_artifacts EXIT TERM
 
-# Copy test binaries on packet server
-echo "### Copying test binaries"
-scp "${SSHOPTS[@]}" /usr/bin/openshift-tests /usr/bin/kubectl "root@${IP}:/usr/local/bin"
-
-# Tests execution
-set +e
-
-# Mirroring test images is supported only for versions greater than or equal to 4.7
-# In such case the dev-scripts private reigstry is reused for mirroring the images.
-# Current openshift version is detected through dev-scripts
-# shellcheck disable=SC2046
-read -d '' OPENSHIFT_VERSION DEVSCRIPTS_REGISTRY DEVSCRIPTS_WORKING_DIR <<<$(ssh "${SSHOPTS[@]}" "root@${IP}" "set +x; source /root/dev-scripts/common.sh; source /root/dev-scripts/ocp_install_env.sh; cd /root/dev-scripts; echo \$(openshift_version); echo \$LOCAL_REGISTRY_DNS_NAME:\$LOCAL_REGISTRY_PORT; echo \$WORKING_DIR")
-
-TEST_ARGS=""
-if printf '%s\n%s' "4.7" "${OPENSHIFT_VERSION}" | sort -C -V; then
-  echo "### Mirroring test images"
-
-  DEVSCRIPTS_TEST_IMAGE_REPO=${DEVSCRIPTS_REGISTRY}/localimages/local-test-image  
-  # shellcheck disable=SC2087
-  ssh "${SSHOPTS[@]}" "root@${IP}" bash - << EOF
-set +x
+function copy_test_binaries() {
+    # Copy test binaries on packet server
+    echo "### Copying test binaries"
+    scp "${SSHOPTS[@]}" /usr/bin/openshift-tests /usr/bin/kubectl "root@${IP}:/usr/local/bin"
+}
 
-source /root/dev-scripts/common.sh
-source /root/dev-scripts/ocp_install_env.sh
+function mirror_test_images() {
+        echo "### Mirroring test images"
 
+        DEVSCRIPTS_TEST_IMAGE_REPO=${DS_REGISTRY}/localimages/local-test-image  
+        # shellcheck disable=SC2087
+        ssh "${SSHOPTS[@]}" "root@${IP}" bash - << EOF
 openshift-tests images --to-repository ${DEVSCRIPTS_TEST_IMAGE_REPO} > /tmp/mirror
-oc image mirror -f /tmp/mirror --registry-config ${DEVSCRIPTS_WORKING_DIR}/pull_secret.json
+oc image mirror -f /tmp/mirror --registry-config ${DS_WORKING_DIR}/pull_secret.json
 EOF
+        TEST_ARGS="--from-repository ${DEVSCRIPTS_TEST_IMAGE_REPO}"
+}
 
-  TEST_ARGS="--from-repository ${DEVSCRIPTS_TEST_IMAGE_REPO}"
-
-  echo "### Enriching test-list cases"
-  cat "${SHARED_DIR}/test-list-ext" >> "${SHARED_DIR}/test-list"
-fi
+function use_minimal_test_list() {
+        echo "### Skipping test images mirroring, fall back to minimal tests list"
+
+        TEST_ARGS="--file /tmp/tests"
+        TEST_SKIPS=""
+        echo "${TEST_MINIMAL_LIST}" > /tmp/tests
+}
 
-# Test upgrade for workflows that requested it
-if [[ "$RUN_UPGRADE_TEST" == true ]]; then
-    echo "### Running Upgrade tests"
-    timeout \
-    --kill-after 10m \
-    120m \
-        ssh \
-            "${SSHOPTS[@]}" \
-            "root@${IP}" \
-            openshift-tests \
-            run-upgrade \
-            ${TEST_ARGS} \
-            --to-image "$OPENSHIFT_UPGRADE_RELEASE_IMAGE" \
-            -o /tmp/artifacts/e2e-upgrade.log \
-            --junit-dir /tmp/artifacts/junit-upgrade \
-            platform
-else
-    if [[ -s "${SHARED_DIR}/test-list" ]]; then
-        echo "### Copying test-list file"
-        scp \
-            "${SSHOPTS[@]}" \
-            "${SHARED_DIR}/test-list" \
-            "root@${IP}:/tmp/test-list"
-        echo "### Running tests"
-        timeout \
-        --kill-after 10m \
-        120m \
-        ssh \
-            "${SSHOPTS[@]}" \
-            "root@${IP}" \
-            openshift-tests \
-            run \
-            "openshift/conformance" \
-            --dry-run \
-            \| grep -Ff /tmp/test-list \|openshift-tests run ${TEST_ARGS} -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit -f -
+case "${CLUSTER_TYPE}" in
+packet)
+    # shellcheck source=/dev/null
+    source "${SHARED_DIR}/packet-conf.sh"
+    # shellcheck source=/dev/null
+    source "${SHARED_DIR}/ds-vars.conf"
+    copy_test_binaries
+    export TEST_PROVIDER=\"\"
+
+    echo "### Checking release version"
+    # Mirroring test images is supported only for versions greater than or equal to 4.7
+    if printf '%s\n%s' "4.7" "${DS_OPENSHIFT_VERSION}" | sort -C -V; then
+        mirror_test_images       
     else
-        echo "### Running tests"
-        ssh \
-            "${SSHOPTS[@]}" \
-            "root@${IP}" \
-            openshift-tests \
-            run \
-            "openshift/conformance/parallel" \
-            --dry-run \
-            \| grep 'Feature:ProjectAPI' \| openshift-tests run ${TEST_ARGS} -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit -f -
+        use_minimal_test_list
     fi
-fi
+    ;;
+*) echo >&2 "Unsupported cluster type '${CLUSTER_TYPE}'"; exit 1;;
+esac
+
+function upgrade() {
+    set -x
+    ssh "${SSHOPTS[@]}" "root@${IP}" \
+        openshift-tests run-upgrade all \
+        --to-image "${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}" \
+        --provider "${TEST_PROVIDER}" \
+        -o "/tmp/artifacts/e2e.log" \
+        --junit-dir "/tmp/artifacts/junit"
+    set +x
+}
 
-rv=$?
+function suite() {
+    if [[ -n "${TEST_SKIPS}" ]]; then
+        TESTS="$(ssh "${SSHOPTS[@]}" "root@${IP}" openshift-tests run --dry-run --provider "${TEST_PROVIDER}" "${TEST_SUITE}")"
+        echo "${TESTS}" | grep -v "${TEST_SKIPS}" >/tmp/tests
+        echo "Skipping tests:"
+        echo "${TESTS}" | grep "${TEST_SKIPS}"
+        TEST_ARGS="${TEST_ARGS:-} --file /tmp/tests"
+    fi
+
+    scp "${SSHOPTS[@]}" /tmp/tests "root@${IP}:/tmp/tests"
+
+    set -x
+    ssh "${SSHOPTS[@]}" "root@${IP}" \
+        openshift-tests run "${TEST_SUITE}" "${TEST_ARGS:-}" \
+        --provider "${TEST_PROVIDER}" \
+        -o "/tmp/artifacts/e2e.log" \
+        --junit-dir "/tmp/artifacts/junit"
+    set +x
+}
 
-set -e
-echo "### Done! (${rv})"
-exit $rv
+echo "$(date +%s)" > "${SHARED_DIR}/TEST_TIME_TEST_START"
+
+case "${TEST_TYPE}" in
+upgrade-conformance)
+    upgrade
+    TEST_LIMIT_START_TIME="$(date +%s)" TEST_SUITE=openshift/conformance/parallel suite
+    ;;
+upgrade)
+    upgrade
+    ;;
+suite-conformance)
+    suite
+    TEST_LIMIT_START_TIME="$(date +%s)" TEST_SUITE=openshift/conformance/parallel suite
+    ;;
+suite)
+    suite
+    ;;
+*)
+    echo >&2 "Unsupported test type '${TEST_TYPE}'"
+    exit 1
+    ;;
+esac