Switch OSAC CI from kustomize to helm-based refresh

ci-operator/config/osac-project/osac-installer/osac-project-osac-installer-main.yaml

@@ -71,7 +71,7 @@ tests:
 - as: e2e-vmaas
   capabilities:
   - intranet
-  run_if_changed: ^(base/|overlays/|scripts/|prerequisites/|charts/)
+  run_if_changed: ^(base/|overlays/|scripts/|prerequisites/|charts/|values/)
   steps:
     dependencies:
       OSAC_INSTALLER_IMAGE: osac-installer

ci-operator/config/osac-project/osac-test-infra/osac-project-osac-test-infra-main.yaml

@@ -71,30 +71,13 @@ tests:
 - as: e2e-vmaas-periodic
   capabilities:
   - intranet
-  cron: 0 6,18 * * *
+  cron: 7 * * * *
   steps:
     workflow: osac-project-cluster-tool-vmaas
-- as: e2e-vmaas-full-setup-kustomize
-  capabilities:
-  - intranet
-  cron: 0 0 * * *
-  steps:
-    cluster_profile: packet-osac
-    env:
-      ASSISTED_CONFIG: |
-        OLM_OPERATORS=cnv,lvm
-        NUM_MASTERS=1
-        NUM_WORKERS=0
-        MASTER_MEMORY=57344
-        MASTER_DISK_COUNT=2
-        MASTER_DISK=200000000000
-        MASTER_CPU=24
-        OPENSHIFT_VERSION=4.20
-    workflow: osac-project-ofcir-baremetal
 - as: e2e-vmaas-full-setup-helm
   capabilities:
   - intranet
-  cron: 0 0 * * *
+  cron: 0 */4 * * *
   steps:
     cluster_profile: packet-osac
     env:

ci-operator/jobs/osac-project/osac-installer/osac-project-osac-installer-main-presubmits.yaml

@@ -17,7 +17,7 @@ presubmits:
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
     name: pull-ci-osac-project-osac-installer-main-e2e-vmaas
     rerun_command: /test e2e-vmaas
-    run_if_changed: ^(base/|overlays/|scripts/|prerequisites/|charts/)
+    run_if_changed: ^(base/|overlays/|scripts/|prerequisites/|charts/|values/)
     spec:
       containers:
       - args:

ci-operator/jobs/osac-project/osac-test-infra/osac-project-osac-test-infra-main-periodics.yaml

@@ -1,7 +1,7 @@
 periodics:
 - agent: kubernetes
   cluster: build03
-  cron: 0 0 * * *
+  cron: 0 */4 * * *
   decorate: true
   decoration_config:
     sparse_checkout_files:
@@ -86,92 +86,7 @@ periodics:
         secretName: result-aggregator
 - agent: kubernetes
   cluster: build03
-  cron: 0 0 * * *
-  decorate: true
-  decoration_config:
-    sparse_checkout_files:
-    - Containerfile
-  extra_refs:
-  - base_ref: main
-    org: osac-project
-    repo: osac-test-infra
-    sparse_checkout_files:
-    - Containerfile
-  labels:
-    capability/intranet: intranet
-    ci-operator.openshift.io/cloud: packet-edge
-    ci-operator.openshift.io/cloud-cluster-profile: packet-osac
-    ci.openshift.io/generator: prowgen
-    job-release: "4.20"
-    pj-rehearse.openshift.io/can-be-rehearsed: "true"
-  name: periodic-ci-osac-project-osac-test-infra-main-e2e-vmaas-full-setup-kustomize
-  spec:
-    containers:
-    - args:
-      - --gcs-upload-secret=/secrets/gcs/service-account.json
-      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
-      - --lease-server-credentials-file=/etc/boskos/credentials
-      - --report-credentials-file=/etc/report/credentials
-      - --secret-dir=/secrets/ci-pull-credentials
-      - --target=e2e-vmaas-full-setup-kustomize
-      command:
-      - ci-operator
-      env:
-      - name: HTTP_SERVER_IP
-        valueFrom:
-          fieldRef:
-            fieldPath: status.podIP
-      image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
-      imagePullPolicy: Always
-      name: ""
-      ports:
-      - containerPort: 8080
-        name: http
-      resources:
-        requests:
-          cpu: 10m
-      volumeMounts:
-      - mountPath: /etc/boskos
-        name: boskos
-        readOnly: true
-      - mountPath: /secrets/ci-pull-credentials
-        name: ci-pull-credentials
-        readOnly: true
-      - mountPath: /secrets/gcs
-        name: gcs-credentials
-        readOnly: true
-      - mountPath: /secrets/manifest-tool
-        name: manifest-tool-local-pusher
-        readOnly: true
-      - mountPath: /etc/pull-secret
-        name: pull-secret
-        readOnly: true
-      - mountPath: /etc/report
-        name: result-aggregator
-        readOnly: true
-    serviceAccountName: ci-operator
-    volumes:
-    - name: boskos
-      secret:
-        items:
-        - key: credentials
-          path: credentials
-        secretName: boskos-credentials
-    - name: ci-pull-credentials
-      secret:
-        secretName: ci-pull-credentials
-    - name: manifest-tool-local-pusher
-      secret:
-        secretName: manifest-tool-local-pusher
-    - name: pull-secret
-      secret:
-        secretName: registry-pull-credentials
-    - name: result-aggregator
-      secret:
-        secretName: result-aggregator
-- agent: kubernetes
-  cluster: build03
-  cron: 0 6,18 * * *
+  cron: 7 * * * *
   decorate: true
   decoration_config:
     sparse_checkout_files:

ci-operator/step-registry/osac-project/cluster-tool/boot/osac-project-cluster-tool-boot-commands.sh

@@ -8,14 +8,14 @@ echo "************ cluster-tool boot ************"
 echo "CLUSTER_TOOL_COMMIT: ${CLUSTER_TOOL_COMMIT}"
 echo "CLUSTER_TOOL_FLAVOR_IMAGE: ${CLUSTER_TOOL_FLAVOR_IMAGE}"
 echo "E2E_NAMESPACE: ${E2E_NAMESPACE}"
-echo "E2E_KUSTOMIZE_OVERLAY: ${E2E_KUSTOMIZE_OVERLAY}"
+echo "E2E_VALUES_FILE: ${E2E_VALUES_FILE}"
 echo "E2E_VM_TEMPLATE: ${E2E_VM_TEMPLATE}"
 echo "OSAC_INSTALLER_IMAGE: ${OSAC_INSTALLER_IMAGE}"
 echo "COMPONENT_IMAGE: ${COMPONENT_IMAGE:-<none>}"
 echo "COMPONENT_IMAGE_NAME: ${COMPONENT_IMAGE_NAME:-<none>}"
 echo "-------------------------------------------"
 
-CLONE_NAME="vmaas-kustomize"
+CLONE_NAME="vmaas-helm"
 
 # === Create ssh_config from ofcir-acquire output ===
 IP=$(cat "${SHARED_DIR}/server-ip")
@@ -102,13 +102,15 @@ COMMIT="$1"
 FLAVOR_IMAGE="$2"
 CLONE="$3"
 INSTALLER_IMAGE="$4"
-KUSTOMIZE_OVERLAY="$5"
+VALUES_FILE="$5"
 VM_TEMPLATE="$6"
 COMPONENT_IMAGE="${7:-}"
 COMPONENT_IMAGE_NAME="${8:-}"
 NAMESPACE="${9:-osac-e2e-ci}"
 AAP_SOURCE_REPO_URL="${10:-}"
 
+VALUES_DIR=$(dirname "${VALUES_FILE}")
+
 _timer() {
     local elapsed=$(( $(date +%s) - $1 ))
     printf "[TIMING] %s: %dm %ds\n" "$2" $((elapsed/60)) $((elapsed%60))
@@ -163,7 +165,7 @@ _timer $PULL_START "Pull flavor"
 
 CLUSTER_BOOT_START=$(date +%s)
 echo "=== Booting cluster ==="
-python3 /usr/local/bin/cluster-tool boot --flavor vmaas-kustomize --name "${CLONE}"
+python3 /usr/local/bin/cluster-tool boot --flavor vmaas-helm --name "${CLONE}"
 _timer $CLUSTER_BOOT_START "Boot cluster"
 
 systemctl restart dnsmasq
@@ -190,8 +192,16 @@ oc set data secret/pull-secret -n openshift-config \
 # --- Phase 4: component override (conditional) ---
 COMPONENT_OVERRIDE_CMD=""
 if [[ -n "${COMPONENT_IMAGE}" ]] && [[ -n "${COMPONENT_IMAGE_NAME}" ]]; then
-    echo "=== Component override: ${COMPONENT_IMAGE_NAME} ==="
-    COMPONENT_OVERRIDE_CMD="curl -fsSL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.6.0/kustomize_v5.6.0_linux_amd64.tar.gz | tar xzf - -C /usr/local/bin && cd /installer/base && kustomize edit set image ${COMPONENT_IMAGE_NAME}=${COMPONENT_IMAGE} && cd /installer && "
+    echo "=== Component override: ${COMPONENT_IMAGE_NAME} → ${COMPONENT_IMAGE} ==="
+    COMPONENT_TAG="${COMPONENT_IMAGE##*:}"
+    COMPONENT_REPO="${COMPONENT_IMAGE%:*}"
+    VF="/installer/${VALUES_FILE}"
+    # Replace full image:tag references — match entire registry/path to handle short names (e.g. osac-aap)
+    COMPONENT_OVERRIDE_CMD="sed -i 's|[a-zA-Z0-9./_-]*${COMPONENT_IMAGE_NAME}:[a-zA-Z0-9._-]*|${COMPONENT_IMAGE}|g' ${VF} && "
+    # Replace split repository/tag format (e.g. operator.image.repository + operator.image.tag)
+    COMPONENT_OVERRIDE_CMD="${COMPONENT_OVERRIDE_CMD}sed -i '/repository: .*${COMPONENT_IMAGE_NAME##*/}/{s|repository: .*|repository: ${COMPONENT_REPO}|;n;s/tag: .*/tag: ${COMPONENT_TAG}/}' ${VF} && "
+    # Verify the override landed
+    COMPONENT_OVERRIDE_CMD="${COMPONENT_OVERRIDE_CMD}grep -q '${COMPONENT_TAG}' ${VF} || { echo 'ERROR: component image override not found in values file'; exit 1; } && "
 fi
 
 # When testing an osac-aap PR, the installer-with-pr image contains
@@ -202,14 +212,15 @@ AAP_OVERRIDE_CMD=""
 AAP_SOURCE_SHA=$(podman run --authfile /root/pull-secret --rm "${INSTALLER_IMAGE}" cat /installer/.aap-source-sha 2>/dev/null || true)
 if [[ -n "${AAP_SOURCE_SHA}" ]]; then
     echo "=== AAP project git ref override: ${AAP_SOURCE_SHA} ==="
-    AAP_OVERRIDE_CMD="sed -i 's|AAP_PROJECT_GIT_BRANCH=.*|AAP_PROJECT_GIT_BRANCH=${AAP_SOURCE_SHA}|' /installer/overlays/${KUSTOMIZE_OVERLAY}/kustomization.yaml && grep -q 'AAP_PROJECT_GIT_BRANCH=${AAP_SOURCE_SHA}' /installer/overlays/${KUSTOMIZE_OVERLAY}/kustomization.yaml || { echo 'ERROR: AAP_PROJECT_GIT_BRANCH override failed'; exit 1; } && "
+    VF="/installer/${VALUES_FILE}"
+    AAP_OVERRIDE_CMD="sed -i 's|projectGitBranch: .*|projectGitBranch: \"${AAP_SOURCE_SHA}\"|' ${VF} && grep -q '${AAP_SOURCE_SHA}' ${VF} || { echo 'ERROR: projectGitBranch override failed'; exit 1; } && "
     if [[ -n "${AAP_SOURCE_REPO_URL}" ]]; then
         echo "=== AAP project git URI override: ${AAP_SOURCE_REPO_URL} ==="
-        AAP_OVERRIDE_CMD="${AAP_OVERRIDE_CMD}sed -i 's|AAP_PROJECT_GIT_URI=.*|AAP_PROJECT_GIT_URI=${AAP_SOURCE_REPO_URL}|' /installer/overlays/${KUSTOMIZE_OVERLAY}/kustomization.yaml && grep -q 'AAP_PROJECT_GIT_URI=${AAP_SOURCE_REPO_URL}' /installer/overlays/${KUSTOMIZE_OVERLAY}/kustomization.yaml || { echo 'ERROR: AAP_PROJECT_GIT_URI override failed'; exit 1; } && "
+        AAP_OVERRIDE_CMD="${AAP_OVERRIDE_CMD}sed -i 's|projectGitUri: .*|projectGitUri: \"${AAP_SOURCE_REPO_URL}\"|' ${VF} && grep -q '${AAP_SOURCE_REPO_URL}' ${VF} || { echo 'ERROR: projectGitUri override failed'; exit 1; } && "
     fi
     if [[ -n "${COMPONENT_IMAGE}" ]]; then
         echo "=== AAP EE image override: ${COMPONENT_IMAGE} ==="
-        AAP_OVERRIDE_CMD="${AAP_OVERRIDE_CMD}sed -i 's|AAP_EE_IMAGE=.*|AAP_EE_IMAGE=${COMPONENT_IMAGE}|' /installer/overlays/${KUSTOMIZE_OVERLAY}/kustomization.yaml && grep -q 'AAP_EE_IMAGE=${COMPONENT_IMAGE}' /installer/overlays/${KUSTOMIZE_OVERLAY}/kustomization.yaml || { echo 'ERROR: AAP_EE_IMAGE override failed'; exit 1; } && "
+        AAP_OVERRIDE_CMD="${AAP_OVERRIDE_CMD}sed -i 's|eeImage: .*|eeImage: \"${COMPONENT_IMAGE}\"|' ${VF} && grep -q 'eeImage:.*${COMPONENT_IMAGE}' ${VF} || { echo 'ERROR: eeImage override failed'; exit 1; } && "
     fi
 fi
 
@@ -218,14 +229,14 @@ REFRESH_START=$(date +%s)
 echo "=== Running refresh ==="
 podman run --authfile /root/pull-secret --rm --network=host \
     -v "${KUBECONFIG_PATH}":/root/.kube/config:z \
-    -v /root/pull-secret:/installer/overlays/${KUSTOMIZE_OVERLAY}/files/quay-pull-secret.json:z \
-    -v /tmp/license.zip:/installer/overlays/${KUSTOMIZE_OVERLAY}/files/license.zip:z \
+    -v /tmp/license.zip:/installer/${VALUES_DIR}/license.zip:z \
+    -v /root/pull-secret:/installer/${VALUES_DIR}/pull-secret.json:z \
     -e KUBECONFIG=/root/.kube/config \
-    -e INSTALLER_KUSTOMIZE_OVERLAY="${KUSTOMIZE_OVERLAY}" \
+    -e VALUES_FILE="${VALUES_FILE}" \
     -e INSTALLER_VM_TEMPLATE="${VM_TEMPLATE}" \
     -e INSTALLER_NAMESPACE="${NAMESPACE}" \
     "${INSTALLER_IMAGE}" \
-    bash -c "${COMPONENT_OVERRIDE_CMD}${AAP_OVERRIDE_CMD}cd /installer && sh scripts/refresh-after-snapshot.sh"
+    bash -c "${COMPONENT_OVERRIDE_CMD}${AAP_OVERRIDE_CMD}cd /installer && python3 -u scripts/refresh-after-snapshot.py"
 _timer $REFRESH_START "Refresh"
 
 echo ""
@@ -240,7 +251,7 @@ timeout -s 9 50m ssh -F "${SHARED_DIR}/ssh_config" ci_machine \
     '${CLUSTER_TOOL_FLAVOR_IMAGE}' \
     '${CLONE_NAME}' \
     '${OSAC_INSTALLER_IMAGE}' \
-    '${E2E_KUSTOMIZE_OVERLAY}' \
+    '${E2E_VALUES_FILE}' \
     '${E2E_VM_TEMPLATE}' \
     '${COMPONENT_IMAGE:-}' \
     '${COMPONENT_IMAGE_NAME:-}' \

ci-operator/step-registry/osac-project/cluster-tool/boot/osac-project-cluster-tool-boot-ref.yaml

@@ -21,9 +21,9 @@ ref:
   - name: E2E_NAMESPACE
     default: "osac-e2e-ci"
     documentation: The namespace to use for OSAC deployment
-  - name: E2E_KUSTOMIZE_OVERLAY
-    default: "vmaas-ci"
-    documentation: The kustomize overlay to use
+  - name: E2E_VALUES_FILE
+    default: "values/vmaas-ci/values.yaml"
+    documentation: Helm values file path (relative to installer root)
   - name: E2E_VM_TEMPLATE
     default: "osac.templates.ocp_virt_vm"
     documentation: The VM template to publish
@@ -34,9 +34,9 @@ ref:
     default: "main"
     documentation: cluster-tool git ref to download (branch, tag, or commit)
   - name: CLUSTER_TOOL_FLAVOR_IMAGE
-    default: "quay.io/rh-ee-ovishlit/cluster-flavors:vmaas-kustomize"
+    default: "quay.io/rh-ee-ovishlit/cluster-flavors:vmaas-helm"
     documentation: OCI image containing the OSAC snapshot flavor
   documentation: |-
     Boots an OSAC cluster from a pre-built snapshot using cluster-tool,
-    then refreshes domain-sensitive resources. Optionally overrides a
-    component image for PR testing.
+    then refreshes domain-sensitive resources via helm upgrade.
+    Optionally overrides a component image for PR testing.

ci-operator/step-registry/osac-project/cluster-tool/destroy/osac-project-cluster-tool-destroy-commands.sh

@@ -5,7 +5,7 @@ set -o pipefail
 
 echo "************ cluster-tool destroy ************"
 
-CLONE_NAME="vmaas-kustomize"
+CLONE_NAME="vmaas-helm"
 
 timeout -s 9 5m ssh -F "${SHARED_DIR}/ssh_config" ci_machine bash -s "${CLONE_NAME}" <<'REMOTE_EOF' || true
 set -o pipefail

ci-operator/step-registry/osac-project/cluster-tool/test/osac-project-cluster-tool-test-commands.sh

@@ -11,7 +11,7 @@ echo "E2E_NAMESPACE: ${E2E_NAMESPACE}"
 echo "E2E_VM_TEMPLATE: ${E2E_VM_TEMPLATE}"
 echo "-------------------------------------------"
 
-CLONE_NAME="vmaas-kustomize"
+CLONE_NAME="vmaas-helm"
 KUBECONFIG_PATH="/root/.kube/${CLONE_NAME}.kubeconfig"
 REMOTE_RESULTS_DIR="/tmp/test-results"
 

ci-operator/step-registry/osac-project/gather/osac-project-gather-commands.sh

@@ -87,6 +87,10 @@ for ns in ${VM_NAMESPACES}; do
     oc get pvc -n "${ns}" -o wide > "${ARTIFACT_DIR}/cnv/${ns}/pvcs.txt" 2>&1 || true
     oc get events -n "${ns}" --sort-by=.lastTimestamp > "${ARTIFACT_DIR}/cnv/${ns}/events.txt" 2>&1 || true
     oc get networkpolicies -n "${ns}" -o yaml > "${ARTIFACT_DIR}/cnv/${ns}/networkpolicies.yaml" 2>&1 || true
+    oc get pods -n "${ns}" -o wide > "${ARTIFACT_DIR}/cnv/${ns}/pods.txt" 2>&1 || true
+    for pod in $(oc get pods -n "${ns}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do
+        oc logs "${pod}" -n "${ns}" --all-containers > "${ARTIFACT_DIR}/cnv/${ns}/pod-${pod}.log" 2>&1 || true
+    done
 done
 
 echo "=== Collecting compute instance status ==="
@@ -117,6 +121,31 @@ echo "=== Collecting cluster operator status ==="
 oc get co > "${ARTIFACT_DIR}/clusteroperators.txt" 2>&1 || true
 oc get csv -n openshift-cnv -o wide > "${ARTIFACT_DIR}/cnv/csv.txt" 2>&1 || true
 
+echo "=== Collecting storage diagnostics ==="
+mkdir -p "${ARTIFACT_DIR}/storage"
+oc get pods -n openshift-storage -o wide > "${ARTIFACT_DIR}/storage/pods.txt" 2>&1 || true
+oc get events -n openshift-storage --sort-by=.lastTimestamp > "${ARTIFACT_DIR}/storage/events.txt" 2>&1 || true
+oc get lvmcluster -n openshift-storage -o yaml > "${ARTIFACT_DIR}/storage/lvmcluster.yaml" 2>&1 || true
+oc get lvmvolumegroups -n openshift-storage -o yaml > "${ARTIFACT_DIR}/storage/lvmvolumegroups.yaml" 2>&1 || true
+oc get sc -o wide > "${ARTIFACT_DIR}/storage/storageclasses.txt" 2>&1 || true
+oc get pv -o wide > "${ARTIFACT_DIR}/storage/pvs.txt" 2>&1 || true
+oc get pvc -A -o wide > "${ARTIFACT_DIR}/storage/pvcs-all.txt" 2>&1 || true
+oc get volumeattachments -o wide > "${ARTIFACT_DIR}/storage/volumeattachments.txt" 2>&1 || true
+for pod in $(oc get pods -n openshift-storage -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do
+    oc logs "${pod}" -n openshift-storage > "${ARTIFACT_DIR}/storage/pod-${pod}.log" 2>&1 || true
+done
+
+echo "=== Collecting MachineConfig diagnostics ==="
+mkdir -p "${ARTIFACT_DIR}/mco"
+oc get mcp -o wide > "${ARTIFACT_DIR}/mco/mcp.txt" 2>&1 || true
+oc get mc --sort-by=.metadata.creationTimestamp > "${ARTIFACT_DIR}/mco/mc.txt" 2>&1 || true
+oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null \
+    | base64 -d 2>/dev/null | jq -r '.auths | keys[]' > "${ARTIFACT_DIR}/mco/pull-secret-registries.txt" 2>&1 || true
+
+echo "=== Collecting service account pull secret state ==="
+oc get sa -n "${E2E_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/serviceaccounts.yaml" 2>&1 || true
+oc get secrets -n "${E2E_NAMESPACE}" -o custom-columns='NAME:.metadata.name,TYPE:.type' > "${ARTIFACT_DIR}/secrets-types.txt" 2>&1 || true
+
 echo "=== Collecting AAP operator status ==="
 oc get ansibleautomationplatform -n "${E2E_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/aap-status.yaml" 2>&1 || true
 oc get automationcontroller -n "${E2E_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/automationcontroller-status.yaml" 2>&1 || true