From 8a3b0cb47a62e88761e8edded22442ff0a7246b3 Mon Sep 17 00:00:00 2001 From: Juan Manuel Parrilla Madrid Date: Wed, 10 Jun 2026 15:18:02 +0100 Subject: [PATCH] ci(hypershift): resolve N-minor release images via release controller API Replace static ci-operator release imports (n1minor through n4minor) with runtime resolution via the release controller REST API. This eliminates concurrent import pods that caused race conditions in ci-tools, failing ~40-45% of e2e runs before tests executed. Changes: - Remove n1minor..n4minor and unused latest-418/419/420 release definitions - Replace OCP_IMAGE_NX dependencies with NODEPOOL_NX_VERSION env vars - Add hypershift-resolve-nodepool-releases step to all 8 e2e workflows - Add nounset-safe variable expansion (${OCP_IMAGE_NX:-}) in run-e2e scripts to prevent crashes when variables are unset - Improve resolve script with retry logic, HTTP status checks, and hard failure on resolution errors Fixes the permafail introduced by #79627 where unguarded variable expansion under set -o nounset crashed run-e2e scripts before tests could execute. Jira: CNTRLPLANE-3509 Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Juan Manuel Parrilla Madrid --- .../hypershift/openshift-hypershift-main.yaml | 90 +++++------------- .../hypershift-aws-e2e-external-workflow.yaml | 1 + ...rshift-aws-e2e-external-oidc-workflow.yaml | 1 + .../hypershift-aws-e2e-metrics-workflow.yaml | 1 + .../hypershift-aws-e2e-nested-workflow.yaml | 3 +- ...ypershift-aws-run-e2e-external-commands.sh | 13 ++- .../hypershift-aws-run-e2e-nested-commands.sh | 12 ++- .../hypershift-azure-aks-e2e-workflow.yaml | 3 +- ...hift-azure-aks-external-oidc-workflow.yaml | 3 +- ...zure-create-selfmanaged-guests-commands.sh | 4 + ...t-azure-create-selfmanaged-guests-ref.yaml | 2 +- ...shift-azure-e2e-self-managed-workflow.yaml | 1 + ...ft-azure-e2e-v2-self-managed-workflow.yaml | 1 + ...ift-azure-run-e2e-self-managed-commands.sh | 8 +- .../hypershift-azure-run-e2e-commands.sh | 12 ++- .../resolve-nodepool-releases/OWNERS | 12 +++ ...hift-resolve-nodepool-releases-commands.sh | 92 +++++++++++++++++++ ...esolve-nodepool-releases-ref.metadata.json | 19 ++++ ...ershift-resolve-nodepool-releases-ref.yaml | 34 +++++++ 19 files changed, 228 insertions(+), 84 deletions(-) create mode 100644 ci-operator/step-registry/hypershift/resolve-nodepool-releases/OWNERS create mode 100755 ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-commands.sh create mode 100644 ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-ref.metadata.json create mode 100644 ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-ref.yaml diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml index 6acf730c43ddb..e025802138e7e 100644 --- a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml @@ -77,21 +77,6 @@ releases: include_built_images: true name: "5.0" namespace: ocp - latest-418: - candidate: - product: ocp - stream: ci - version: "4.18" - latest-419: - candidate: - product: ocp - stream: ci - version: "4.19" - latest-420: - candidate: - product: ocp - stream: ci - version: "4.20" latest-421: candidate: product: ocp @@ -115,26 +100,6 @@ releases: product: ocp stream: nightly version: "5.0" - n1minor: - candidate: - product: ocp - stream: ci - version: "4.22" - n2minor: - candidate: - product: ocp - stream: ci - version: "4.21" - n3minor: - candidate: - product: ocp - stream: ci - version: "4.20" - n4minor: - candidate: - product: ocp - stream: ci - version: "4.19" resources: '*': limits: @@ -156,11 +121,6 @@ tests: pipeline_skip_if_only_changed: (^(\.tekton|\.github|\.claude|docs|examples|enhancements|contrib|\.cursor|test/envtest)/)|(\.md$)|((^|/)OWNERS$)|(/overrides\.yaml$)|(^renovate\.json$)|(/\.testcoverage\.yml$)|(^\.gitlint$)|(^\.gitignore$)|(^\.coderabbit\.yaml$)|(^\.dockerignore$)|(^codecov\.yml$) steps: cluster_profile: hypershift-aks - dependencies: - OCP_IMAGE_N1: release:n1minor - OCP_IMAGE_N2: release:n2minor - OCP_IMAGE_N3: release:n3minor - OCP_IMAGE_N4: release:n4minor env: AUTH_THROUGH_CERTS: "true" ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" @@ -169,6 +129,10 @@ tests: HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER: "" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_SKU: "" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION: "" + NODEPOOL_N1_VERSION: "4.22" + NODEPOOL_N2_VERSION: "4.21" + NODEPOOL_N3_VERSION: "4.20" + NODEPOOL_N4_VERSION: "4.19" post: - ref: hypershift-analyze-e2e-failure - chain: cucushift-installer-rehearse-azure-aks-deprovision @@ -180,15 +144,15 @@ tests: cluster_profile: hypershift-aks dependencies: OCP_IMAGE_LATEST: release:latest-422 - OCP_IMAGE_N1: release:latest-421 - OCP_IMAGE_N2: release:latest-420 - OCP_IMAGE_N3: release:latest-419 - OCP_IMAGE_N4: release:latest-418 OCP_IMAGE_PREVIOUS: release:initial-422 env: AUTH_THROUGH_CERTS: "true" ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" HYPERSHIFT_AZURE_LOCATION: centralus + NODEPOOL_N1_VERSION: "4.21" + NODEPOOL_N2_VERSION: "4.20" + NODEPOOL_N3_VERSION: "4.19" + NODEPOOL_N4_VERSION: "4.18" workflow: hypershift-azure-aks-e2e - as: e2e-aks-override run_if_changed: (/overrides\.yaml$) @@ -207,13 +171,12 @@ tests: pipeline_skip_if_only_changed: (^(\.tekton|\.github|\.claude|docs|examples|enhancements|contrib|\.cursor|test/envtest)/)|(\.md$)|((^|/)OWNERS$)|(/overrides\.yaml$)|(^renovate\.json$)|(/\.testcoverage\.yml$)|(^\.gitlint$)|(^\.gitignore$)|(^\.coderabbit\.yaml$)|(^\.dockerignore$)|(^codecov\.yml$) steps: cluster_profile: hypershift-aws - dependencies: - OCP_IMAGE_N1: release:n1minor - OCP_IMAGE_N2: release:n2minor - OCP_IMAGE_N3: release:n3minor - OCP_IMAGE_N4: release:n4minor env: ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" + NODEPOOL_N1_VERSION: "4.22" + NODEPOOL_N2_VERSION: "4.21" + NODEPOOL_N3_VERSION: "4.20" + NODEPOOL_N4_VERSION: "4.19" REQUEST_SERVING_COMPONENT_TEST: "true" post: - ref: hypershift-analyze-e2e-failure @@ -283,25 +246,23 @@ tests: pipeline_skip_if_only_changed: (^(\.tekton|\.github|\.claude|docs|examples|enhancements|contrib|\.cursor|test/envtest)/)|(\.md$)|((^|/)OWNERS$)|(/overrides\.yaml$)|(^renovate\.json$)|(/\.testcoverage\.yml$)|(^\.gitlint$)|(^\.gitignore$)|(^\.coderabbit\.yaml$)|(^\.dockerignore$)|(^codecov\.yml$) steps: cluster_profile: hypershift-azure - dependencies: - OCP_IMAGE_N1: release:n1minor env: ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" HYPERSHIFT_AZURE_LOCATION: centralus + NODEPOOL_N1_VERSION: "4.22" workflow: hypershift-azure-e2e-v2-self-managed - always_run: false as: e2e-aws-minimal optional: true steps: cluster_profile: hypershift-aws - dependencies: - OCP_IMAGE_N1: release:n1minor - OCP_IMAGE_N2: release:n2minor - OCP_IMAGE_N3: release:n3minor - OCP_IMAGE_N4: release:n4minor env: CI_TESTS_RUN: (^(TestCreateCluster$|TestCreateClusterV2$)) ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" + NODEPOOL_N1_VERSION: "4.22" + NODEPOOL_N2_VERSION: "4.21" + NODEPOOL_N3_VERSION: "4.20" + NODEPOOL_N4_VERSION: "4.19" REQUEST_SERVING_COMPONENT_TEST: "true" workflow: hypershift-aws-e2e-nested - always_run: false @@ -309,13 +270,12 @@ tests: optional: true steps: cluster_profile: hypershift-aws - dependencies: - OCP_IMAGE_N1: release:n1minor - OCP_IMAGE_N2: release:n2minor - OCP_IMAGE_N3: release:n3minor - OCP_IMAGE_N4: release:n4minor env: ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" + NODEPOOL_N1_VERSION: "4.22" + NODEPOOL_N2_VERSION: "4.21" + NODEPOOL_N3_VERSION: "4.20" + NODEPOOL_N4_VERSION: "4.19" REQUEST_SERVING_COMPONENT_TEST: "true" TECH_PREVIEW_NO_UPGRADE: "true" workflow: hypershift-aws-e2e-nested @@ -353,13 +313,13 @@ tests: cluster_profile: hypershift-aws dependencies: OCP_IMAGE_LATEST: release:latest-422 - OCP_IMAGE_N1: release:latest-421 - OCP_IMAGE_N2: release:latest-420 - OCP_IMAGE_N3: release:latest-419 - OCP_IMAGE_N4: release:latest-418 OCP_IMAGE_PREVIOUS: release:initial-422 env: ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" + NODEPOOL_N1_VERSION: "4.21" + NODEPOOL_N2_VERSION: "4.20" + NODEPOOL_N3_VERSION: "4.19" + NODEPOOL_N4_VERSION: "4.18" REQUEST_SERVING_COMPONENT_TEST: "true" RUN_KARPENTER_TESTS: "true" workflow: hypershift-aws-e2e-nested diff --git a/ci-operator/step-registry/hypershift/aws/e2e/external/hypershift-aws-e2e-external-workflow.yaml b/ci-operator/step-registry/hypershift/aws/e2e/external/hypershift-aws-e2e-external-workflow.yaml index a5db8ba0b7341..e2963243b8700 100644 --- a/ci-operator/step-registry/hypershift/aws/e2e/external/hypershift-aws-e2e-external-workflow.yaml +++ b/ci-operator/step-registry/hypershift/aws/e2e/external/hypershift-aws-e2e-external-workflow.yaml @@ -21,3 +21,4 @@ workflow: pre: - ref: ipi-install-rbac - chain: hypershift-setup-root-management-cluster + - ref: hypershift-resolve-nodepool-releases diff --git a/ci-operator/step-registry/hypershift/aws/e2e/external/oidc/hypershift-aws-e2e-external-oidc-workflow.yaml b/ci-operator/step-registry/hypershift/aws/e2e/external/oidc/hypershift-aws-e2e-external-oidc-workflow.yaml index fb3a2bb77ae5c..b4a4a75efe5ef 100644 --- a/ci-operator/step-registry/hypershift/aws/e2e/external/oidc/hypershift-aws-e2e-external-oidc-workflow.yaml +++ b/ci-operator/step-registry/hypershift/aws/e2e/external/oidc/hypershift-aws-e2e-external-oidc-workflow.yaml @@ -16,3 +16,4 @@ workflow: - chain: hypershift-setup-nested-management-cluster - ref: idp-external-oidc-keycloak-server - ref: hypershift-install + - ref: hypershift-resolve-nodepool-releases diff --git a/ci-operator/step-registry/hypershift/aws/e2e/metrics/hypershift-aws-e2e-metrics-workflow.yaml b/ci-operator/step-registry/hypershift/aws/e2e/metrics/hypershift-aws-e2e-metrics-workflow.yaml index 9a86aafedc58e..72439d25b2a71 100644 --- a/ci-operator/step-registry/hypershift/aws/e2e/metrics/hypershift-aws-e2e-metrics-workflow.yaml +++ b/ci-operator/step-registry/hypershift/aws/e2e/metrics/hypershift-aws-e2e-metrics-workflow.yaml @@ -34,3 +34,4 @@ workflow: - chain: hypershift-setup-nested-management-cluster - ref: hypershift-install - ref: hypershift-install-metrics + - ref: hypershift-resolve-nodepool-releases diff --git a/ci-operator/step-registry/hypershift/aws/e2e/nested/hypershift-aws-e2e-nested-workflow.yaml b/ci-operator/step-registry/hypershift/aws/e2e/nested/hypershift-aws-e2e-nested-workflow.yaml index 8123328063c3b..fb648ab54adea 100644 --- a/ci-operator/step-registry/hypershift/aws/e2e/nested/hypershift-aws-e2e-nested-workflow.yaml +++ b/ci-operator/step-registry/hypershift/aws/e2e/nested/hypershift-aws-e2e-nested-workflow.yaml @@ -31,4 +31,5 @@ workflow: pre: - ref: ipi-install-rbac - chain: hypershift-setup-nested-management-cluster - - ref: hypershift-install \ No newline at end of file + - ref: hypershift-install + - ref: hypershift-resolve-nodepool-releases \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/aws/run-e2e/external/hypershift-aws-run-e2e-external-commands.sh b/ci-operator/step-registry/hypershift/aws/run-e2e/external/hypershift-aws-run-e2e-external-commands.sh index c5dda28472898..db3b27d78ac11 100644 --- a/ci-operator/step-registry/hypershift/aws/run-e2e/external/hypershift-aws-run-e2e-external-commands.sh +++ b/ci-operator/step-registry/hypershift/aws/run-e2e/external/hypershift-aws-run-e2e-external-commands.sh @@ -43,27 +43,30 @@ if [[ "${AWS_MULTI_ARCH:-}" == "true" ]]; then AWS_MULTI_ARCH_PARAMS="--e2e.aws-multi-arch=true" fi +if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then + source "${SHARED_DIR}/nodepool_release_images" +fi + N1_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N1} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N1:-}" && "${OCP_IMAGE_N1}" != "${OCP_IMAGE_LATEST}" ]]; then N1_NP_VERSION_TEST_ARGS="--e2e.n1-minor-release-image=${OCP_IMAGE_N1}" fi N2_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N2} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N2:-}" && "${OCP_IMAGE_N2}" != "${OCP_IMAGE_LATEST}" ]]; then N2_NP_VERSION_TEST_ARGS="--e2e.n2-minor-release-image=${OCP_IMAGE_N2}" fi N3_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N3} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N3:-}" && "${OCP_IMAGE_N3}" != "${OCP_IMAGE_LATEST}" ]]; then N3_NP_VERSION_TEST_ARGS="--e2e.n3-minor-release-image=${OCP_IMAGE_N3}" fi N4_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N4} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N4:-}" && "${OCP_IMAGE_N4}" != "${OCP_IMAGE_LATEST}" ]]; then N4_NP_VERSION_TEST_ARGS="--e2e.n4-minor-release-image=${OCP_IMAGE_N4}" fi - ADDITIONAL_PULL_SECRET_PARAMS="" if check_e2e_flag 'e2e.additional-pull-secret-file' && [[ -f /etc/hypershift-additional-pull-secret/.dockerconfigjson ]]; then ADDITIONAL_PULL_SECRET_PARAMS="--e2e.additional-pull-secret-file=/etc/hypershift-additional-pull-secret/.dockerconfigjson" diff --git a/ci-operator/step-registry/hypershift/aws/run-e2e/nested/hypershift-aws-run-e2e-nested-commands.sh b/ci-operator/step-registry/hypershift/aws/run-e2e/nested/hypershift-aws-run-e2e-nested-commands.sh index 3758fe686bcb3..f95342faa53bb 100644 --- a/ci-operator/step-registry/hypershift/aws/run-e2e/nested/hypershift-aws-run-e2e-nested-commands.sh +++ b/ci-operator/step-registry/hypershift/aws/run-e2e/nested/hypershift-aws-run-e2e-nested-commands.sh @@ -43,23 +43,27 @@ if [[ "${AWS_MULTI_ARCH:-}" == "true" ]]; then AWS_MULTI_ARCH_PARAMS="--e2e.aws-multi-arch=true" fi +if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then + source "${SHARED_DIR}/nodepool_release_images" +fi + N1_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N1} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N1:-}" && "${OCP_IMAGE_N1}" != "${OCP_IMAGE_LATEST}" ]]; then N1_NP_VERSION_TEST_ARGS="--e2e.n1-minor-release-image=${OCP_IMAGE_N1}" fi N2_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N2} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N2:-}" && "${OCP_IMAGE_N2}" != "${OCP_IMAGE_LATEST}" ]]; then N2_NP_VERSION_TEST_ARGS="--e2e.n2-minor-release-image=${OCP_IMAGE_N2}" fi N3_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N3} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N3:-}" && "${OCP_IMAGE_N3}" != "${OCP_IMAGE_LATEST}" ]]; then N3_NP_VERSION_TEST_ARGS="--e2e.n3-minor-release-image=${OCP_IMAGE_N3}" fi N4_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N4} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N4:-}" && "${OCP_IMAGE_N4}" != "${OCP_IMAGE_LATEST}" ]]; then N4_NP_VERSION_TEST_ARGS="--e2e.n4-minor-release-image=${OCP_IMAGE_N4}" fi diff --git a/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml b/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml index 0d31e063e36e6..ae475917694f2 100644 --- a/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml +++ b/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml @@ -23,7 +23,8 @@ workflow: - chain: cucushift-installer-rehearse-azure-aks-provision - ref: hypershift-azure-aks-attach-kv - ref: hypershift-install - - ref: cucushift-hypershift-extended-k8s-mgmt-get-guest-annotations + - ref: cucushift-hypershift-extended-k8s-mgmt-get-guest-annotations + - ref: hypershift-resolve-nodepool-releases test: - ref: hypershift-azure-run-e2e post: diff --git a/ci-operator/step-registry/hypershift/azure/aks/external-oidc/hypershift-azure-aks-external-oidc-workflow.yaml b/ci-operator/step-registry/hypershift/azure/aks/external-oidc/hypershift-azure-aks-external-oidc-workflow.yaml index ada75c7ea7e0a..681415d928378 100644 --- a/ci-operator/step-registry/hypershift/azure/aks/external-oidc/hypershift-azure-aks-external-oidc-workflow.yaml +++ b/ci-operator/step-registry/hypershift/azure/aks/external-oidc/hypershift-azure-aks-external-oidc-workflow.yaml @@ -10,7 +10,8 @@ workflow: - ref: idp-external-oidc-keycloak-aks-server - ref: hypershift-azure-aks-attach-kv - ref: hypershift-install - - ref: cucushift-hypershift-extended-k8s-mgmt-get-guest-annotations + - ref: cucushift-hypershift-extended-k8s-mgmt-get-guest-annotations + - ref: hypershift-resolve-nodepool-releases test: - ref: hypershift-azure-run-e2e post: diff --git a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh index 974487d10fc0f..0341fb58a9d52 100644 --- a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh @@ -6,4 +6,8 @@ set -exuo pipefail export KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" export HYPERSHIFT_BINARY="${HYPERSHIFT_BINARY:-/hypershift/bin/hypershift}" +if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then + source "${SHARED_DIR}/nodepool_release_images" +fi + /hypershift/bin/create-guests diff --git a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml index d7e665080d3bf..2db583e9eebf3 100644 --- a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml +++ b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml @@ -15,7 +15,7 @@ ref: dependencies: - name: "release:latest" env: RELEASE_IMAGE_LATEST - - name: "release:n1minor" + - name: "release:latest" env: OCP_IMAGE_N1 env: - name: HYPERSHIFT_NODE_COUNT diff --git a/ci-operator/step-registry/hypershift/azure/e2e/self-managed/hypershift-azure-e2e-self-managed-workflow.yaml b/ci-operator/step-registry/hypershift/azure/e2e/self-managed/hypershift-azure-e2e-self-managed-workflow.yaml index 43cd9e22d3e55..6384948b3f0d1 100644 --- a/ci-operator/step-registry/hypershift/azure/e2e/self-managed/hypershift-azure-e2e-self-managed-workflow.yaml +++ b/ci-operator/step-registry/hypershift/azure/e2e/self-managed/hypershift-azure-e2e-self-managed-workflow.yaml @@ -29,6 +29,7 @@ workflow: - chain: hypershift-setup-nested-management-cluster - ref: hypershift-azure-setup-private-link - ref: hypershift-install + - ref: hypershift-resolve-nodepool-releases env: CLOUD_PROVIDER: "Azure" HYPERSHIFT_NODE_COUNT: "2" diff --git a/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml b/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml index 8fc3b3d822ec3..bad47d829cea0 100644 --- a/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml +++ b/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml @@ -24,6 +24,7 @@ workflow: - chain: hypershift-setup-nested-management-cluster - ref: hypershift-azure-setup-private-link - ref: hypershift-install + - ref: hypershift-resolve-nodepool-releases - chain: hypershift-azure-create-selfmanaged-guests test: - chain: hypershift-azure-run-e2e-v2-selfmanaged diff --git a/ci-operator/step-registry/hypershift/azure/run-e2e-self-managed/hypershift-azure-run-e2e-self-managed-commands.sh b/ci-operator/step-registry/hypershift/azure/run-e2e-self-managed/hypershift-azure-run-e2e-self-managed-commands.sh index 1061bf11d2469..e259f7d6954cb 100644 --- a/ci-operator/step-registry/hypershift/azure/run-e2e-self-managed/hypershift-azure-run-e2e-self-managed-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/run-e2e-self-managed/hypershift-azure-run-e2e-self-managed-commands.sh @@ -36,13 +36,17 @@ check_e2e_flag() { return $? } +if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then + source "${SHARED_DIR}/nodepool_release_images" +fi + N1_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N1} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N1:-}" && "${OCP_IMAGE_N1}" != "${OCP_IMAGE_LATEST}" ]]; then N1_NP_VERSION_TEST_ARGS="--e2e.n1-minor-release-image=${OCP_IMAGE_N1}" fi N2_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N2} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N2:-}" && "${OCP_IMAGE_N2}" != "${OCP_IMAGE_LATEST}" ]]; then N2_NP_VERSION_TEST_ARGS="--e2e.n2-minor-release-image=${OCP_IMAGE_N2}" fi diff --git a/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh b/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh index 25bdb73490ae0..fe9fc4ccf7cf4 100644 --- a/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh @@ -67,23 +67,27 @@ if [[ -n "$HYPERSHIFT_MANAGED_SERVICE" ]]; then export MANAGED_SERVICE="$HYPERSHIFT_MANAGED_SERVICE" fi +if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then + source "${SHARED_DIR}/nodepool_release_images" +fi + N1_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N1} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N1:-}" && "${OCP_IMAGE_N1}" != "${OCP_IMAGE_LATEST}" ]]; then N1_NP_VERSION_TEST_ARGS="--e2e.n1-minor-release-image=${OCP_IMAGE_N1}" fi N2_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N2} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N2:-}" && "${OCP_IMAGE_N2}" != "${OCP_IMAGE_LATEST}" ]]; then N2_NP_VERSION_TEST_ARGS="--e2e.n2-minor-release-image=${OCP_IMAGE_N2}" fi N3_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N3} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N3:-}" && "${OCP_IMAGE_N3}" != "${OCP_IMAGE_LATEST}" ]]; then N3_NP_VERSION_TEST_ARGS="--e2e.n3-minor-release-image=${OCP_IMAGE_N3}" fi N4_NP_VERSION_TEST_ARGS="" -if [[ ${OCP_IMAGE_N4} != "${OCP_IMAGE_LATEST}" ]]; then +if [[ -n "${OCP_IMAGE_N4:-}" && "${OCP_IMAGE_N4}" != "${OCP_IMAGE_LATEST}" ]]; then N4_NP_VERSION_TEST_ARGS="--e2e.n4-minor-release-image=${OCP_IMAGE_N4}" fi diff --git a/ci-operator/step-registry/hypershift/resolve-nodepool-releases/OWNERS b/ci-operator/step-registry/hypershift/resolve-nodepool-releases/OWNERS new file mode 100644 index 0000000000000..4ce99ba551da4 --- /dev/null +++ b/ci-operator/step-registry/hypershift/resolve-nodepool-releases/OWNERS @@ -0,0 +1,12 @@ +approvers: + - jparrill + - bryan-cox + - csrwng + - sjenning + - enxebre +reviewers: + - jparrill + - bryan-cox + - csrwng + - sjenning + - enxebre diff --git a/ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-commands.sh b/ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-commands.sh new file mode 100755 index 0000000000000..a5450dc86a1ec --- /dev/null +++ b/ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-commands.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail +set -o xtrace + +RELEASE_CONTROLLER_URL="https://amd64.ocp.releases.ci.openshift.org" +REGISTRY_AUTH="/etc/ci-pull-credentials/.dockerconfigjson" +STREAMS=(ci nightly) + +resolve_from_stream() { + local version=$1 + local stream=$2 + local url="${RELEASE_CONTROLLER_URL}/api/v1/releasestream/${version}.0-0.${stream}/latest" + local response http_code body pullspec + + for attempt in 1 2 3; do + response=$(curl -q -L -s -w "\n%{http_code}" --retry 3 --retry-delay 5 --connect-timeout 10 --max-time 30 "${url}") + http_code=$(echo "${response}" | tail -1) + body=$(echo "${response}" | sed '$d') + + if [[ "${http_code}" == "200" ]]; then + pullspec=$(echo "${body}" | jq -r ".pullSpec // empty") + if [[ -n "${pullspec}" ]]; then + echo "${pullspec}" + return 0 + fi + fi + echo "Attempt ${attempt}/3 failed for ${version} stream=${stream} (HTTP ${http_code}), retrying..." >&2 + sleep $((attempt * 5)) + done + return 1 +} + +verify_image_pullable() { + local pullspec=$1 + if [[ -f "${REGISTRY_AUTH}" ]]; then + oc image info --filter-by-os linux/amd64 -a "${REGISTRY_AUTH}" "${pullspec}" &>/dev/null + else + oc image info --filter-by-os linux/amd64 "${pullspec}" &>/dev/null + fi +} + +resolve_release_image() { + local version=$1 + if [[ -z "${version}" ]]; then + return 1 + fi + + for stream in "${STREAMS[@]}"; do + local pullspec + if pullspec=$(resolve_from_stream "${version}" "${stream}"); then + if verify_image_pullable "${pullspec}"; then + echo "${pullspec}" + return 0 + fi + echo "WARNING: ${version} stream=${stream} resolved to ${pullspec} but image is not available in registry" >&2 + else + echo "WARNING: ${version} stream=${stream} has no Accepted release" >&2 + fi + done + + echo "ERROR: Failed to resolve a pullable release image for version ${version} across streams: ${STREAMS[*]}" >&2 + return 1 +} + +OUTPUT_FILE="${SHARED_DIR}/nodepool_release_images" +: > "${OUTPUT_FILE}" + +failures=0 +for n in 1 2 3 4; do + version_var="NODEPOOL_N${n}_VERSION" + version="${!version_var:-}" + if [[ -n "${version}" ]]; then + if resolved=$(resolve_release_image "${version}"); then + echo "export OCP_IMAGE_N${n}=\"${resolved}\"" >> "${OUTPUT_FILE}" + echo "Resolved N${n} (${version}): ${resolved}" + else + echo "ERROR: Could not resolve N${n} version ${version}" + failures=$((failures + 1)) + fi + fi +done + +echo "--- Resolved nodepool release images ---" +cat "${OUTPUT_FILE}" + +if [[ ${failures} -gt 0 ]]; then + echo "ERROR: Failed to resolve ${failures} release image(s)." + exit 1 +fi diff --git a/ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-ref.metadata.json b/ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-ref.metadata.json new file mode 100644 index 0000000000000..4e3f19e6a6d96 --- /dev/null +++ b/ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-ref.metadata.json @@ -0,0 +1,19 @@ +{ + "path": "hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-ref.yaml", + "owners": { + "approvers": [ + "jparrill", + "bryan-cox", + "csrwng", + "sjenning", + "enxebre" + ], + "reviewers": [ + "jparrill", + "bryan-cox", + "csrwng", + "sjenning", + "enxebre" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-ref.yaml b/ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-ref.yaml new file mode 100644 index 0000000000000..e995e2d40e72e --- /dev/null +++ b/ci-operator/step-registry/hypershift/resolve-nodepool-releases/hypershift-resolve-nodepool-releases-ref.yaml @@ -0,0 +1,34 @@ +ref: + as: hypershift-resolve-nodepool-releases + cli: latest + commands: hypershift-resolve-nodepool-releases-commands.sh + credentials: + - mount_path: /etc/ci-pull-credentials + name: ci-pull-credentials + namespace: test-credentials + env: + - default: "" + name: NODEPOOL_N1_VERSION + documentation: "OCP minor version for n-1 NodePool skew test (e.g. 4.22). Resolved via release controller API." + - default: "" + name: NODEPOOL_N2_VERSION + documentation: "OCP minor version for n-2 NodePool skew test (e.g. 4.21). Resolved via release controller API." + - default: "" + name: NODEPOOL_N3_VERSION + documentation: "OCP minor version for n-3 NodePool skew test (e.g. 4.20). Resolved via release controller API." + - default: "" + name: NODEPOOL_N4_VERSION + documentation: "OCP minor version for n-4 NodePool skew test (e.g. 4.19). Resolved via release controller API." + from: hypershift-tests + grace_period: 5m0s + resources: + requests: + cpu: 100m + memory: 100Mi + timeout: 10m0s + documentation: |- + Resolves N-minor OCP release images via the release controller API and + writes them to ${SHARED_DIR}/nodepool_release_images for consumption by + downstream run-e2e steps. Verifies each resolved image is available in the + registry before accepting it. Falls back to the nightly stream when CI + stream images have been garbage-collected.