diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml index 6acf730c43ddb..7cb81f613d56e 100644 --- a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml @@ -481,6 +481,9 @@ tests: leases: - env: OPENSTACK_CLOUD resource_type: openstack-vexxhost-quota-slice + post: + - ref: hypershift-openstack-gather + - chain: hypershift-destroy-nested-management-cluster test: - chain: hypershift-openstack-e2e workflow: hypershift-aws-e2e-nested diff --git a/ci-operator/step-registry/hypershift/openstack/e2e/execute/hypershift-openstack-e2e-execute-commands.sh b/ci-operator/step-registry/hypershift/openstack/e2e/execute/hypershift-openstack-e2e-execute-commands.sh index b52029f3cf4d0..52bbeaffaa96b 100755 --- a/ci-operator/step-registry/hypershift/openstack/e2e/execute/hypershift-openstack-e2e-execute-commands.sh +++ b/ci-operator/step-registry/hypershift/openstack/e2e/execute/hypershift-openstack-e2e-execute-commands.sh @@ -48,4 +48,4 @@ hack/ci-test-e2e.sh ${E2E_EXTRA_ARGS} \ --e2e.openstack-credentials-file="${SHARED_DIR}/clouds.yaml" \ --e2e.openstack-external-network-id="${OPENSTACK_EXTERNAL_NETWORK_ID}" \ --e2e.openstack-node-flavor="${OPENSTACK_COMPUTE_FLAVOR}" \ - --e2e.openstack-node-image-name="${RHCOS_IMAGE_NAME}" + ${RHCOS_IMAGE_NAME:+--e2e.openstack-node-image-name="${RHCOS_IMAGE_NAME}"} diff --git a/ci-operator/step-registry/hypershift/openstack/e2e/execute/hypershift-openstack-e2e-execute-ref.yaml b/ci-operator/step-registry/hypershift/openstack/e2e/execute/hypershift-openstack-e2e-execute-ref.yaml index a94b175f313a5..7a73b19cd8663 100644 --- a/ci-operator/step-registry/hypershift/openstack/e2e/execute/hypershift-openstack-e2e-execute-ref.yaml +++ b/ci-operator/step-registry/hypershift/openstack/e2e/execute/hypershift-openstack-e2e-execute-ref.yaml @@ -15,7 +15,15 @@ ref: - env: OCP_IMAGE_PREVIOUS name: release:initial env: - - default: "rhcos-latest-hcp-nodepool" + - default: "" + documentation: |- + Name of a pre-existing Glance image to use for OpenStack nodes. On 4.19+ + this is unused for TestCreateCluster and TestNodePool since ORC manages + the image lifecycle directly; it is only consumed by TestOpenStackAdvancedTest + (not in the default E2E_TESTS_REGEX). On 4.18 and earlier, DefaultOpenStackOptions + still wired this value into the NodePool ImageName field, so the 4.18 job + configs override this variable to reference a versioned pre-existing image + (e.g. rhcos-4.18-hcp-nodepool). name: RHCOS_IMAGE_NAME - default: 'TestCreateCluster$|TestNodePool|TestAutoscaling' documentation: |- diff --git a/ci-operator/step-registry/hypershift/openstack/gather/OWNERS b/ci-operator/step-registry/hypershift/openstack/gather/OWNERS new file mode 100644 index 0000000000000..f47a23f53a78f --- /dev/null +++ b/ci-operator/step-registry/hypershift/openstack/gather/OWNERS @@ -0,0 +1,4 @@ +approvers: +- openstack-approvers +reviewers: +- openstack-reviewers diff --git a/ci-operator/step-registry/hypershift/openstack/gather/hypershift-openstack-gather-commands.sh b/ci-operator/step-registry/hypershift/openstack/gather/hypershift-openstack-gather-commands.sh new file mode 100644 index 0000000000000..b52d9cf772a4b --- /dev/null +++ b/ci-operator/step-registry/hypershift/openstack/gather/hypershift-openstack-gather-commands.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -euo pipefail + +if [[ -z "${KUBECONFIG:-}" || ! -f "${KUBECONFIG}" ]]; then + echo "No kubeconfig found at ${KUBECONFIG:-}, skipping gather" + exit 0 +fi + +GATHER_DIR="${ARTIFACT_DIR}/capo-gather" +mkdir -p "${GATHER_DIR}" + +echo "Gathering CAPO and ORC resources from management cluster..." + +# Namespaces stuck in Terminating — these are the first sign of a stuck deletion. +echo "=== Namespaces in Terminating state ===" +oc get namespace --field-selector='status.phase=Terminating' -o yaml 2>/dev/null \ + > "${GATHER_DIR}/terminating-namespaces.yaml" || true +oc get namespace --field-selector='status.phase=Terminating' 2>/dev/null || true + +# HostedClusters and NodePools for context. +for resource in hostedclusters nodepools; do + echo "=== ${resource} ===" + oc get "${resource}" -A -o yaml 2>/dev/null \ + > "${GATHER_DIR}/${resource}.yaml" || true +done + +# CAPI core resources (Cluster, Machine) to trace the deletion chain. +for resource in \ + "clusters.cluster.x-k8s.io" \ + "machines.cluster.x-k8s.io"; do + echo "=== ${resource} ===" + name="${resource%%.*}" + oc get "${resource}" -A -o yaml 2>/dev/null \ + > "${GATHER_DIR}/${name}.yaml" || true +done + +# CAPO-specific resources. OpenStackServer carries the CAPO-managed finalizer +# (openstackserver.infrastructure.cluster.x-k8s.io) that can get permanently +# stuck if the CAPO pod is killed before the finalizer is removed. +for resource in \ + "openstackclusters.infrastructure.cluster.x-k8s.io" \ + "openstackmachines.infrastructure.cluster.x-k8s.io" \ + "openstackservers.infrastructure.cluster.x-k8s.io"; do + echo "=== ${resource} ===" + name="${resource%%.*}" + oc get "${resource}" -A -o yaml 2>/dev/null \ + > "${GATHER_DIR}/${name}.yaml" || true +done + +# ORC Image resources. These also carry an in-namespace finalizer (managed by +# the ORC controller) that can block namespace termination. +echo "=== images.openstack.k-orc.cloud ===" +oc get images.openstack.k-orc.cloud -A -o yaml 2>/dev/null \ + > "${GATHER_DIR}/orc-images.yaml" || true + +echo "Done gathering CAPO resources" diff --git a/ci-operator/step-registry/hypershift/openstack/gather/hypershift-openstack-gather-ref.metadata.json b/ci-operator/step-registry/hypershift/openstack/gather/hypershift-openstack-gather-ref.metadata.json new file mode 100644 index 0000000000000..a145562259304 --- /dev/null +++ b/ci-operator/step-registry/hypershift/openstack/gather/hypershift-openstack-gather-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "hypershift/openstack/gather/hypershift-openstack-gather-ref.yaml", + "owners": { + "approvers": [ + "openstack-approvers" + ], + "reviewers": [ + "openstack-reviewers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/openstack/gather/hypershift-openstack-gather-ref.yaml b/ci-operator/step-registry/hypershift/openstack/gather/hypershift-openstack-gather-ref.yaml new file mode 100644 index 0000000000000..8305b25ff6a21 --- /dev/null +++ b/ci-operator/step-registry/hypershift/openstack/gather/hypershift-openstack-gather-ref.yaml @@ -0,0 +1,17 @@ +ref: + as: hypershift-openstack-gather + best_effort: true + cli: latest + commands: hypershift-openstack-gather-commands.sh + from: upi-installer + optional_on_success: true + resources: + requests: + cpu: 10m + memory: 100Mi + timeout: 5m0s + documentation: |- + Gather CAPO (OpenStackServer, OpenStackMachine, OpenStackCluster) and ORC + Image resources from the management cluster. Runs after e2e test failures to + capture the state of any objects whose finalizers may have blocked namespace + termination and caused the test to time out.