Test metal ipi upgrade

ci-operator/config/openshift-metal3/dev-scripts/openshift-metal3-dev-scripts-master.yaml

@@ -31,6 +31,15 @@ tests:
         IP_STACK=v4v6
         NETWORK_TYPE=OVNKubernetes
     workflow: baremetalds-e2e
+- as: e2e-metal-ipi-ocp-sdn-ipv4-upgrade
+  steps:
+    cluster_profile: packet
+    env:
+      DEVSCRIPTS_CONFIG: |
+        IP_STACK=v4
+        NETWORK_TYPE=OpenShiftSDN
+      RUN_UPGRADE_TEST: "true"
+    workflow: baremetalds-e2e-upgrade
 zz_generated_metadata:
   branch: master
   org: openshift-metal3

ci-operator/jobs/openshift-metal3/dev-scripts/openshift-metal3-dev-scripts-master-presubmits.yaml

@@ -66,7 +66,72 @@ presubmits:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )e2e-metal-ipi,?($|\s.*)
   - agent: kubernetes
-    always_run: false
+    always_run: true
+    branches:
+    - master
+    cluster: build02
+    context: ci/prow/e2e-metal-ipi-ocp-sdn-ipv4-upgrade
+    decorate: true
+    decoration_config:
+      skip_cloning: true
+    labels:
+      ci-operator.openshift.io/prowgen-controlled: "true"
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-metal3-dev-scripts-master-e2e-metal-ipi-ocp-sdn-ipv4-upgrade
+    rerun_command: /test e2e-metal-ipi-ocp-sdn-ipv4-upgrade
+    spec:
+      containers:
+      - args:
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-password-file=/etc/boskos/password
+        - --report-password-file=/etc/report/password.txt
+        - --report-username=ci
+        - --secret-dir=/usr/local/e2e-metal-ipi-ocp-sdn-ipv4-upgrade-cluster-profile
+        - --target=e2e-metal-ipi-ocp-sdn-ipv4-upgrade
+        command:
+        - ci-operator
+        image: ci-operator:latest
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /usr/local/e2e-metal-ipi-ocp-sdn-ipv4-upgrade-cluster-profile
+          name: cluster-profile
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: password
+            path: password
+          secretName: boskos-credentials
+      - name: cluster-profile
+        projected:
+          sources:
+          - secret:
+              name: cluster-secrets-packet
+          - configMap:
+              name: cluster-profile-packet
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )e2e-metal-ipi-ocp-sdn-ipv4-upgrade,?($|\s.*)
+  - agent: kubernetes
+    always_run: true
     branches:
     - master
     cluster: build02
@@ -78,7 +143,6 @@ presubmits:
       ci-operator.openshift.io/prowgen-controlled: "true"
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
     name: pull-ci-openshift-metal3-dev-scripts-master-e2e-metal-ipi-ovn-dualstack
-    optional: true
     rerun_command: /test e2e-metal-ipi-ovn-dualstack
     spec:
       containers:

ci-operator/step-registry/baremetalds/devscripts/setup/baremetalds-devscripts-setup-commands.sh

@@ -91,7 +91,7 @@ cp /root/pull-secret /root/dev-scripts/pull_secret.json
 
 curl https://mirror.openshift.com/pub/openshift-v4/clients/oc/4.4/linux/oc.tar.gz | tar -C /usr/bin -xzf -
 
-echo "export OPENSHIFT_RELEASE_IMAGE=${RELEASE_IMAGE_LATEST}" >> /root/dev-scripts/config_root.sh
+echo "export OPENSHIFT_RELEASE_IMAGE=${OPENSHIFT_INSTALL_RELEASE_IMAGE}" >> /root/dev-scripts/config_root.sh
 echo "export ADDN_DNS=\$(awk '/nameserver/ { print \$2;exit; }' /etc/resolv.conf)" >> /root/dev-scripts/config_root.sh
 echo "export OPENSHIFT_CI=true" >> /root/dev-scripts/config_root.sh
 echo "export WORKER_MEMORY=16384" >> /root/dev-scripts/config_root.sh

ci-operator/step-registry/baremetalds/devscripts/setup/baremetalds-devscripts-setup-ref.yaml

@@ -10,5 +10,8 @@ ref:
   - name: DEVSCRIPTS_CONFIG
     default: ""
     documentation: Additional dev-scripts configuration, appended to the one defined by the step command. See https://github.com/openshift-metal3/dev-scripts/blob/master/README.md for more details about supported values.
+  dependencies:
+  - name: "release:latest"
+    env: OPENSHIFT_INSTALL_RELEASE_IMAGE
   documentation: |-
     Checkout and execute dev-scripts on a packet server

ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-commands.sh

@@ -3,6 +3,7 @@
 set -o nounset
 set -o errexit
 set -o pipefail
+set -x
 
 echo "************ baremetalds test command ************"
 
@@ -28,25 +29,64 @@ IP=$(cat "${SHARED_DIR}/server-ip")
 
 SSHOPTS=(-o 'ConnectTimeout=5' -o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null' -o 'ServerAliveInterval=90' -i "${CLUSTER_PROFILE_DIR}/.packet-kni-ssh-privatekey")
 
+collect_artifacts() {
+    echo "### Fetching results"
+    ssh "${SSHOPTS[@]}" "root@${IP}" tar -czf - /tmp/artifacts | tar -C "${ARTIFACT_DIR}" -xzf -
+}
+trap collect_artifacts EXIT
+
 # Copy test binaries on packet server
 echo "### Copying test binaries"
 scp "${SSHOPTS[@]}" /usr/bin/openshift-tests /usr/bin/kubectl "root@${IP}:/usr/local/bin"
 
 # Tests execution
 set +e
-if [[ -s "${SHARED_DIR}/test-list" ]]; then
-    echo "### Copying test-list file"
-    scp "${SSHOPTS[@]}" "${SHARED_DIR}/test-list" "root@${IP}:/tmp/test-list"
-    echo "### Running tests"
-    ssh "${SSHOPTS[@]}" "root@${IP}" openshift-tests run "openshift/conformance/parallel" --dry-run \| grep -Ff /tmp/test-list \|openshift-tests run -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit -f -
+
+# Test upgrade for workflows that requested it
+if [[ "$RUN_UPGRADE_TEST" == true ]]; then
+    echo "### Running Upgrade tests"
+    timeout \
+    --kill-after 10m \
+    120m \
+        ssh \
+            "${SSHOPTS[@]}" \
+            "root@${IP}" \
+            openshift-tests \
+            run-upgrade \
+            --to-image "$OPENSHIFT_UPGRADE_RELEASE_IMAGE" \
+            -o /tmp/artifacts/e2e-upgrade.log \
+            --junit-dir /tmp/artifacts/junit-upgrade \
+            platform
 else
-    echo "### Running tests"
-    ssh "${SSHOPTS[@]}" "root@${IP}" openshift-tests run "openshift/conformance/parallel" --dry-run \| grep 'Feature:ProjectAPI' \| openshift-tests run -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit -f -
+    if [[ -s "${SHARED_DIR}/test-list" ]]; then
+        echo "### Copying test-list file"
+        scp \
+            "${SSHOPTS[@]}" \
+            "${SHARED_DIR}/test-list" \
+            "root@${IP}:/tmp/test-list"
+        echo "### Running tests"
+        ssh \
+            "${SSHOPTS[@]}" \
+            "root@${IP}" \
+            openshift-tests \
+            run \
+            "openshift/conformance/parallel" \
+            --dry-run \
+            \| grep -Ff /tmp/test-list \|openshift-tests run -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit -f -
+    else
+        echo "### Running tests"
+        ssh \
+            "${SSHOPTS[@]}" \
+            "root@${IP}" \
+            openshift-tests \
+            run \
+            "openshift/conformance/parallel" \
+            --dry-run \
+            \| grep 'Feature:ProjectAPI' \| openshift-tests run -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit -f -
+    fi
 fi
-rv=$?
 
-echo "### Fetching results"
-ssh "${SSHOPTS[@]}" "root@${IP}" tar -czf - /tmp/artifacts | tar -C "${ARTIFACT_DIR}" -xzf -
+rv=$?
 
 set -e
 echo "### Done! (${rv})"

ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-ref.yaml

@@ -8,5 +8,12 @@ ref:
       memory: 600Mi
     limits:
       memory: 2Gi
+  env:
+  - name: RUN_UPGRADE_TEST
+    default: "false"
+    documentation: If the value is equal to "true", run only the upgrade tests
+  dependencies:
+  - name: "release:latest"
+    env: OPENSHIFT_UPGRADE_RELEASE_IMAGE
   documentation: |-
     The Baremtal DS E2E step executes the common end-to-end test suite.

ci-operator/step-registry/baremetalds/e2e/upgrade/OWNERS

@@ -0,0 +1,7 @@
+approvers:
+- eisraeli
+- andfasano
+- akiselev1
+- derekhiggins
+- honza
+- stbenjam

ci-operator/step-registry/baremetalds/e2e/upgrade/baremetalds-e2e-upgrade-workflow.metadata.json

@@ -0,0 +1,13 @@
+{
+	"path": "baremetalds/e2e/upgrade/baremetalds-e2e-upgrade-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"eisraeli",
+			"andfasano",
+			"akiselev1",
+			"derekhiggins",
+			"honza",
+			"stbenjam"
+		]
+	}
+}

ci-operator/step-registry/baremetalds/e2e/upgrade/baremetalds-e2e-upgrade-workflow.yaml

@@ -0,0 +1,20 @@
+workflow:
+  as: baremetalds-e2e-upgrade
+  steps:
+    allow_skip_on_success: true 
+    pre:
+    - ref: baremetalds-packet-setup
+    - ref: ipi-install-rbac
+    - ref: baremetalds-devscripts-setup
+    test:
+    - ref: baremetalds-e2e-conf
+    - ref: baremetalds-e2e-test
+    post:
+    - ref: baremetalds-devscripts-gather
+    - ref: baremetalds-packet-teardown
+    env:
+      RUN_UPGRADE_TEST: "true"
+    dependencies:
+      OPENSHIFT_INSTALL_RELEASE_IMAGE: release:initial
+  documentation: |-
+    This workflow executes the common end-to-end upgrade test suite on a cluster provisioned by running dev-scripts on a packet server.