diff --git a/Makefile.core.mk b/Makefile.core.mk index a269ee4195..ce0704793d 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -215,8 +215,8 @@ test.integration: envtest ## Run integration tests located in the tests/integrat go run github.com/onsi/ginkgo/v2/ginkgo --tags=integration $(GINKGO_FLAGS) ./tests/integration/... .PHONY: test.scorecard -test.scorecard: operator-sdk ## Run the operator scorecard test. - OPERATOR_SDK=$(OPERATOR_SDK) ${SOURCE_DIR}/tests/scorecard-test.sh +test.scorecard: operator-sdk ## Run the operator scorecard test. Use OCP=true to run against an existing OCP cluster instead of Kind. + OCP=$${OCP:-false} OPERATOR_SDK=$(OPERATOR_SDK) SCORECARD_NAMESPACE="$${SCORECARD_NAMESPACE:-scorecard-test}" ${SOURCE_DIR}/tests/scorecard-test.sh .PHONY: test.e2e.ocp test.e2e.ocp: istioctl ## Run the end-to-end tests against an existing OCP cluster. While running on OCP in downstream you need to set ISTIOCTL_DOWNLOAD_URL to the URL where the istioctl productized binary. diff --git a/tests/e2e/common-operator-integ-suite.sh b/tests/e2e/common-operator-integ-suite.sh index abf4c75c2c..2e06a772c3 100755 --- a/tests/e2e/common-operator-integ-suite.sh +++ b/tests/e2e/common-operator-integ-suite.sh @@ -124,6 +124,17 @@ initialize_variables() { IP_FAMILY=${IP_FAMILY:-ipv4} ISTIO_MANIFEST="chart/samples/istio-sample.yaml" CI=${CI:-"false"} + USE_INTERNAL_REGISTRY=${USE_INTERNAL_REGISTRY:-"false"} + + # Debug logging and fallback for GINKGO_FLAGS + echo "CI environment: ${CI}" + echo "GINKGO_FLAGS received: '${GINKGO_FLAGS:-}'" + + # Fallback: Generate GINKGO_FLAGS if empty and CI=true + if [ -z "${GINKGO_FLAGS:-}" ] && [ "${CI}" == "true" ]; then + GINKGO_FLAGS="--no-color" + echo "Generated GINKGO_FLAGS fallback: '${GINKGO_FLAGS}'" + fi # export to be sure that the variables are available in the subshell export IMAGE_BASE="${IMAGE_BASE:-sail-operator}" @@ -133,22 +144,32 @@ initialize_variables() { # Handle OCP registry scenarios # Note: Makefile.core.mk sets HUB=quay.io/sail-dev and TAG=1.29-latest by default if [ "${OCP}" == "true" ]; then + # Debug output for troubleshooting + echo "DEBUG: CI='${CI}', HUB='${HUB}'" + if [ "${CI}" == "true" ] && [ "${HUB}" == "quay.io/sail-dev" ]; then # Scenario 2: CI mode with default HUB -> use external registry with proper CI tag echo "CI mode detected for OCP, using external registry ${HUB}" - + export USE_INTERNAL_REGISTRY="false" # Use PR_NUMBER if available, otherwise generate timestamp tag + # Use TARGET_ARCH to differentiate tags for different architectures in CI, avoid race conditions in CI when multiple runs are pushing to the same default tag if [ -n "${PR_NUMBER:-}" ]; then - export TAG="pr-${PR_NUMBER}" + TAG="pr-${PR_NUMBER}-${TARGET_ARCH}" + export TAG echo "Using PR-based tag: ${TAG}" else - TAG="ci-test-$(date +%s)" + TAG="ci-test-$(date +%s)-${TARGET_ARCH}" export TAG echo "Using timestamp-based tag: ${TAG}" fi + elif [ "${CI}" == "true" ]; then + # Additional CI mode check - handle CI mode regardless of HUB value + echo "CI mode detected for OCP with custom HUB (${HUB}), using external registry" + export USE_INTERNAL_REGISTRY="false" elif [ "${HUB}" != "quay.io/sail-dev" ]; then # Scenario 3: Custom registry provided by user echo "Using custom registry: ${HUB}" + export USE_INTERNAL_REGISTRY="false" else # Scenario 1: Local development -> use internal OCP registry echo "Local development mode, will use OCP internal registry" @@ -303,7 +324,7 @@ if [ "${SKIP_BUILD}" == "false" ]; then fi fi -export SKIP_DEPLOY IP_FAMILY ISTIO_MANIFEST NAMESPACE CONTROL_PLANE_NS DEPLOYMENT_NAME MULTICLUSTER ARTIFACTS ISTIO_NAME COMMAND KUBECONFIG ISTIOCTL_PATH +export SKIP_DEPLOY IP_FAMILY ISTIO_MANIFEST NAMESPACE CONTROL_PLANE_NS DEPLOYMENT_NAME MULTICLUSTER ARTIFACTS ISTIO_NAME COMMAND KUBECONFIG ISTIOCTL_PATH GINKGO_FLAGS if [ "${OLM}" != "true" ] && [ "${SKIP_DEPLOY}" != "true" ]; then # shellcheck disable=SC2153 diff --git a/tests/e2e/integ-suite-kind.sh b/tests/e2e/integ-suite-kind.sh index f796ad3bd4..29cd3427c7 100755 --- a/tests/e2e/integ-suite-kind.sh +++ b/tests/e2e/integ-suite-kind.sh @@ -35,10 +35,10 @@ function check_prerequisites() { function run_integration_tests() { echo "Running integration tests" if [ "${MULTICLUSTER}" == "true" ]; then - ARTIFACTS="${ARTIFACTS}" ISTIOCTL="${ISTIOCTL}" "${ROOT}/tests/e2e/common-operator-integ-suite.sh" --kind --multicluster + ARTIFACTS="${ARTIFACTS}" ISTIOCTL="${ISTIOCTL}" GINKGO_FLAGS="${GINKGO_FLAGS}" "${ROOT}/tests/e2e/common-operator-integ-suite.sh" --kind --multicluster else KUBECONFIG="${ARTIFACTS}/config" - ARTIFACTS="${ARTIFACTS}" IP_FAMILY="${IP_FAMILY}" "${ROOT}/tests/e2e/common-operator-integ-suite.sh" --kind + ARTIFACTS="${ARTIFACTS}" IP_FAMILY="${IP_FAMILY}" GINKGO_FLAGS="${GINKGO_FLAGS}" "${ROOT}/tests/e2e/common-operator-integ-suite.sh" --kind fi } diff --git a/tests/e2e/integ-suite-ocp.sh b/tests/e2e/integ-suite-ocp.sh index 83566be1a2..a075dd2692 100755 --- a/tests/e2e/integ-suite-ocp.sh +++ b/tests/e2e/integ-suite-ocp.sh @@ -27,4 +27,4 @@ if [ -z "${KUBECONFIG}" ]; then exit 1 fi -KUBECONFIG="${KUBECONFIG}" ./tests/e2e/common-operator-integ-suite.sh --ocp \ No newline at end of file +KUBECONFIG="${KUBECONFIG}" GINKGO_FLAGS="${GINKGO_FLAGS}" ./tests/e2e/common-operator-integ-suite.sh --ocp \ No newline at end of file diff --git a/tests/e2e/setup/build-and-push-operator.sh b/tests/e2e/setup/build-and-push-operator.sh index fadc990c83..4be9123fa6 100755 --- a/tests/e2e/setup/build-and-push-operator.sh +++ b/tests/e2e/setup/build-and-push-operator.sh @@ -86,9 +86,12 @@ build_and_push_operator_image() { # Main logic # Only use internal registry for OCP local development (when USE_INTERNAL_REGISTRY is set) +echo "DEBUG: OCP='${OCP}', USE_INTERNAL_REGISTRY='${USE_INTERNAL_REGISTRY:-false}'" if [ "${OCP}" == "true" ] && [ "${USE_INTERNAL_REGISTRY:-false}" == "true" ]; then echo "Setting up OCP internal registry for local development..." get_internal_registry +else + echo "Skipping internal registry setup - using external registry" fi echo "Registry: ${HUB}" diff --git a/tests/scorecard-test.sh b/tests/scorecard-test.sh index 8c11b31595..f4bdc9da4b 100755 --- a/tests/scorecard-test.sh +++ b/tests/scorecard-test.sh @@ -19,22 +19,55 @@ set -eux -o pipefail SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ROOT="$(dirname "${SCRIPTPATH}")" -# shellcheck source=common/scripts/kind_provisioner.sh -source "${ROOT}/common/scripts/kind_provisioner.sh" +# OCP environment variable can be set to "true" to run tests against an existing OCP cluster instead of a Kind cluster +OCP="${OCP:-false}" -# Create a temporary kubeconfig -KUBECONFIG="$(mktemp)" -export KUBECONFIG +if [[ "${OCP}" == "true" ]]; then + echo "Running scorecard tests against existing OCP cluster" -# Create the kind cluster -export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" -export DEFAULT_CLUSTER_YAML="${ROOT}/tests/e2e/setup/config/default.yaml" -export ARTIFACTS="${ARTIFACTS:-$(mktemp -d)}" -export IP_FAMILY="${IP_FAMILY:-ipv4}" -setup_kind_cluster "${KIND_CLUSTER_NAME}" "" "" "true" "true" + # Check if KUBECONFIG is set + if [ -z "${KUBECONFIG:-}" ]; then + echo "KUBECONFIG is not set. oc will not be able to connect to the cluster. Exiting." + exit 1 + fi -kind export kubeconfig --name="${KIND_CLUSTER_NAME}" + # Verify we can connect to the cluster + if ! oc cluster-info > /dev/null 2>&1; then + echo "Cannot connect to OpenShift cluster. Check your KUBECONFIG and cluster access." + exit 1 + fi + + echo "Connected to cluster: $(oc config current-context)" + +else + echo "Running scorecard tests against Kind cluster" + + # shellcheck source=common/scripts/kind_provisioner.sh + source "${ROOT}/common/scripts/kind_provisioner.sh" + + # Create a temporary kubeconfig + KUBECONFIG="$(mktemp)" + export KUBECONFIG + + # Create the kind cluster + export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" + export DEFAULT_CLUSTER_YAML="${ROOT}/tests/e2e/setup/config/default.yaml" + export ARTIFACTS="${ARTIFACTS:-$(mktemp -d)}" + export IP_FAMILY="${IP_FAMILY:-ipv4}" + setup_kind_cluster "${KIND_CLUSTER_NAME}" "" "" "true" "true" + + kind export kubeconfig --name="${KIND_CLUSTER_NAME}" +fi + +# Determine namespace - use scorecard-test for OCP to avoid conflicts with any existing namespaces, and default for Kind since it's a fresh cluster +NAMESPACE="${SCORECARD_NAMESPACE:-default}" +if [[ "${OCP}" == "true" ]]; then + NAMESPACE="${SCORECARD_NAMESPACE:-scorecard-test}" + # Create namespace if it doesn't exist + oc create namespace "${NAMESPACE}" || true +fi # Run the test OPERATOR_SDK="${OPERATOR_SDK:-operator-sdk}" -${OPERATOR_SDK} scorecard --kubeconfig="${KUBECONFIG}" --namespace=default bundle +echo "Running scorecard tests in namespace: ${NAMESPACE}" +${OPERATOR_SDK} scorecard --kubeconfig="${KUBECONFIG}" --namespace="${NAMESPACE}" bundle