From 953fa1188e3e95366d637dfd9f63f1cb5b6ef67c Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 10 Oct 2022 20:28:55 -0700 Subject: [PATCH] ci-operator/config/openshift/release/openshift-release-master__ci-4.11-upgrade-from-stable-4.10: Drop failing rollback jobs Like 2d73374825d (origin/pr/26629) ci-operator/config/openshift/release: Drop failing minor rollback tests, 2022-02-28, #26629), but for the 4.10-to-4.11-to-4.10 rollbacks. This time both the OVN and SDN rollback jobs are perma-failing [1,2], and in both cases the issue is sticking on [3,4]: INFO: cluster upgrade is Progressing: Working towards 4.10.35: 614 of 773 done (79% complete), waiting on openshift-controller-manager with that operator crash-looping on [5,6]: F1010 09:51:56.918590 1 cmd.go:138] open /var/run/configmaps/config/config.yaml: permission denied I haven't dug in more deeply to try to understand that failure, but as 2d73374825d points out: > Since we don't support minor rollbacks, or really rollbacks of any > sort [12], I'm dropping these jobs instead of root-causing the hang. > ... > [12]: https://github.com/openshift/openshift-docs/blame/d4762f0f626a4dddb9d7330e63a3bb6cb73f5bb5/modules/update-upgrading-cli.adoc#L160-L162 Since then, those docs have moved to [7], but the lack of rollback support still stands. [1]: https://testgrid.k8s.io/redhat-openshift-ocp-release-4.11-informing#periodic-ci-openshift-release-master-ci-4.11-upgrade-from-stable-4.10-e2e-aws-ovn-upgrade-rollback [2]: https://testgrid.k8s.io/redhat-openshift-ocp-release-4.11-informing#periodic-ci-openshift-release-master-ci-4.11-upgrade-from-stable-4.10-e2e-aws-upgrade-rollback [3]: https://prow.ci.openshift.org/view/gs/origin-ci-test/logs/periodic-ci-openshift-release-master-ci-4.11-upgrade-from-stable-4.10-e2e-aws-ovn-upgrade-rollback/1579338022623645696 [4]: https://prow.ci.openshift.org/view/gs/origin-ci-test/logs/periodic-ci-openshift-release-master-ci-4.11-upgrade-from-stable-4.10-e2e-aws-upgrade-rollback/1578454440359235584 [5]: https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-ci-4.11-upgrade-from-stable-4.10-e2e-aws-ovn-upgrade-rollback/1579338022623645696/artifacts/e2e-aws-ovn-upgrade-rollback/gather-extra/artifacts/pods/openshift-controller-manager-operator_openshift-controller-manager-operator-7fbc8cc67d-zbrv4_openshift-controller-manager-operator_previous.log [6]: https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-ci-4.11-upgrade-from-stable-4.10-e2e-aws-upgrade-rollback/1578454440359235584/artifacts/e2e-aws-upgrade-rollback/gather-extra/artifacts/pods/openshift-controller-manager-operator_openshift-controller-manager-operator-7fbc8cc67d-s5pwz_openshift-controller-manager-operator_previous.log [7]: https://github.com/openshift/openshift-docs/blob/7f87267bc69d65abd96e6b783100195c6b78549f/updating/updating-troubleshooting.adoc --- ...ter__ci-4.11-upgrade-from-stable-4.10.yaml | 21 --- .../openshift-release-master-periodics.yaml | 154 ------------------ 2 files changed, 175 deletions(-) diff --git a/ci-operator/config/openshift/release/openshift-release-master__ci-4.11-upgrade-from-stable-4.10.yaml b/ci-operator/config/openshift/release/openshift-release-master__ci-4.11-upgrade-from-stable-4.10.yaml index c8081fcd8d12c..f6d384a68fe8f 100644 --- a/ci-operator/config/openshift/release/openshift-release-master__ci-4.11-upgrade-from-stable-4.10.yaml +++ b/ci-operator/config/openshift/release/openshift-release-master__ci-4.11-upgrade-from-stable-4.10.yaml @@ -65,16 +65,6 @@ tests: FAIL_ON_CORE_DUMP: "true" TEST_TYPE: upgrade-conformance workflow: openshift-upgrade-aws-ovn -- as: e2e-aws-ovn-upgrade-rollback - cron: 8 5 */1 * 1-5 - steps: - cluster_profile: aws-2 - env: - BASE_DOMAIN: aws-2.ci.openshift.org - TEST_TYPE: upgrade-conformance - TEST_UPGRADE_OPTIONS: abort-at=99 - workflow: openshift-upgrade-aws-ovn - timeout: 6h0m0s - as: e2e-gcp-ovn-upgrade cluster: build02 interval: 48h @@ -101,17 +91,6 @@ tests: FAIL_ON_CORE_DUMP: "true" TEST_TYPE: upgrade-conformance workflow: openshift-upgrade-azure-ovn -- as: e2e-aws-upgrade-rollback - interval: 48h - steps: - cluster_profile: aws-2 - env: - BASE_DOMAIN: aws-2.ci.openshift.org - DELETE_MC: "false" - TEST_TYPE: upgrade - TEST_UPGRADE_OPTIONS: abort-at=99 - workflow: openshift-upgrade-aws - timeout: 6h0m0s - as: e2e-aws-uwm interval: 48h steps: diff --git a/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml b/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml index 814c728d4d1e4..c1a6cec8cb887 100644 --- a/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml +++ b/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml @@ -6209,83 +6209,6 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator -- agent: kubernetes - cluster: build01 - cron: 8 5 */1 * 1-5 - decorate: true - decoration_config: - skip_cloning: true - timeout: 6h0m0s - extra_refs: - - base_ref: master - org: openshift - repo: release - labels: - ci-operator.openshift.io/cloud: aws - ci-operator.openshift.io/cloud-cluster-profile: aws-2 - ci-operator.openshift.io/variant: ci-4.11-upgrade-from-stable-4.10 - ci.openshift.io/generator: prowgen - ci.openshift.io/no-builds: "true" - job-release: "4.11" - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-openshift-release-master-ci-4.11-upgrade-from-stable-4.10-e2e-aws-ovn-upgrade-rollback - spec: - containers: - - args: - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --secret-dir=/secrets/ci-pull-credentials - - --secret-dir=/usr/local/e2e-aws-ovn-upgrade-rollback-cluster-profile - - --target=e2e-aws-ovn-upgrade-rollback - - --variant=ci-4.11-upgrade-from-stable-4.10 - command: - - ci-operator - image: ci-operator:latest - imagePullPolicy: Always - name: "" - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /secrets/ci-pull-credentials - name: ci-pull-credentials - readOnly: true - - mountPath: /usr/local/e2e-aws-ovn-upgrade-rollback-cluster-profile - name: cluster-profile - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: ci-pull-credentials - secret: - secretName: ci-pull-credentials - - name: cluster-profile - secret: - secretName: cluster-secrets-aws-2 - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator - agent: kubernetes cluster: build01 cron: 17 4 */1 * 1-5 @@ -6438,83 +6361,6 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator -- agent: kubernetes - cluster: build01 - decorate: true - decoration_config: - skip_cloning: true - timeout: 6h0m0s - extra_refs: - - base_ref: master - org: openshift - repo: release - interval: 48h - labels: - ci-operator.openshift.io/cloud: aws - ci-operator.openshift.io/cloud-cluster-profile: aws-2 - ci-operator.openshift.io/variant: ci-4.11-upgrade-from-stable-4.10 - ci.openshift.io/generator: prowgen - ci.openshift.io/no-builds: "true" - job-release: "4.11" - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-openshift-release-master-ci-4.11-upgrade-from-stable-4.10-e2e-aws-upgrade-rollback - spec: - containers: - - args: - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --secret-dir=/secrets/ci-pull-credentials - - --secret-dir=/usr/local/e2e-aws-upgrade-rollback-cluster-profile - - --target=e2e-aws-upgrade-rollback - - --variant=ci-4.11-upgrade-from-stable-4.10 - command: - - ci-operator - image: ci-operator:latest - imagePullPolicy: Always - name: "" - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /secrets/ci-pull-credentials - name: ci-pull-credentials - readOnly: true - - mountPath: /usr/local/e2e-aws-upgrade-rollback-cluster-profile - name: cluster-profile - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: ci-pull-credentials - secret: - secretName: ci-pull-credentials - - name: cluster-profile - secret: - secretName: cluster-secrets-aws-2 - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator - agent: kubernetes cluster: build01 cron: 56 7 */1 * 1-5