Adding template for ipi e2e job

ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml

@@ -1173,6 +1173,70 @@ presubmits:
         secret:
           secretName: sentry-dsn
     trigger: (?m)^/test( | .* )e2e-gcp-upi,?($|\s.*)
+  - agent: kubernetes
+    always_run: false
+    branches:
+    - master
+    context: ci/prow/e2e-ipi
+    decorate: true
+    decoration_config:
+      skip_cloning: true
+    labels:
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-installer-master-e2e-ipi
+    rerun_command: /test e2e-ipi
+    spec:
+      containers:
+      - args:
+        - --artifact-dir=$(ARTIFACTS)
+        - --give-pr-author-access-to-namespace=true
+        - --secret-dir=/usr/local/e2e-ipi-cluster-profile
+        - --sentry-dsn-path=/etc/sentry-dsn/ci-operator
+        - --target=e2e-ipi
+        - --template=/usr/local/e2e-ipi
+        command:
+        - ci-operator
+        env:
+        - name: CLUSTER_TYPE
+          value: metal
+        - name: CONFIG_SPEC
+          valueFrom:
+            configMapKeyRef:
+              key: openshift-installer-master.yaml
+              name: ci-operator-master-configs
+        - name: JOB_NAME_SAFE
+          value: e2e-ipi
+        - name: TEST_COMMAND
+          value: run-remote-tests
+        image: ci-operator:latest
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /usr/local/e2e-ipi-cluster-profile
+          name: cluster-profile
+        - mountPath: /usr/local/e2e-ipi
+          name: job-definition
+          subPath: cluster-launch-installer-e2e.yaml
+        - mountPath: /etc/sentry-dsn
+          name: sentry-dsn
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: cluster-profile
+        projected:
+          sources:
+          - secret:
+              name: cluster-secrets-metal
+      - configMap:
+          name: prow-job-cluster-launch-installer-upi-e2e
+        name: job-definition
+      - name: sentry-dsn
+        secret:
+          secretName: sentry-dsn
+    trigger: (?m)^/test( | .* )e2e-ipi,?($|\s.*)
   - agent: kubernetes
     always_run: false
     branches:

ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml

@@ -11,8 +11,12 @@ parameters:
 - name: IMAGE_FORMAT
 - name: IMAGE_INSTALLER
   required: true
+- name: IMAGE_LIBVIRT_INSTALLER
+  required: true
 - name: IMAGE_TESTS
   required: true
+- name: IMAGE_UPI_INSTALLER
+  required: true
 - name: CLUSTER_TYPE
   required: true
 - name: TEST_COMMAND
@@ -97,7 +101,7 @@ objects:
       # we want to gather the teardown logs no matter what
       ci-operator.openshift.io/wait-for-container-artifacts: teardown
       ci-operator.openshift.io/save-container-logs: "true"
-      ci-operator.openshift.io/container-sub-tests: "lease,setup,test,teardown"
+      ci-operator.openshift.io/container-sub-tests: "lease,setup,test,teardown,setup-packet"
   spec:
     restartPolicy: Never
     activeDeadlineSeconds: 18000
@@ -141,6 +145,12 @@ objects:
         #!/bin/bash
         set -euo pipefail
 
+        # TODO(derekh): New metal type added to boskos.ci ???
+        if [ "${CLUSTER_TYPE}" == "metal" ] ; then
+            touch /tmp/shared/leased
+            exit 0
+        fi
+
         trap 'rc=$?; CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi; if test "${rc}" -ne 0; then touch /tmp/shared/exit; fi; exit "${rc}"' EXIT
 
         # hack for bazel
@@ -212,7 +222,7 @@ objects:
       - -c
       - |
         #!/bin/bash
-        set -euo pipefail
+        set -exuo pipefail
 
         export PATH=/usr/libexec/origin:$PATH
 
@@ -329,6 +339,13 @@ objects:
           export KUBE_SSH_USER=core
         elif [[ "${CLUSTER_TYPE}" == "azure4" ]]; then
           export TEST_PROVIDER='azure'
+        elif [ "${CLUSTER_TYPE}" == "metal" ] ; then
+          export HOME=/tmp/shared/nss_wrapper
+          export NSS_WRAPPER_PASSWD=$HOME/passwd NSS_WRAPPER_GROUP=$HOME/group NSS_USERNAME=nsswrapper NSS_GROUPNAME=nsswrapper LD_PRELOAD=/tmp/shared/libnss_wrapper.so
+          export IP=$(cat /tmp/shared/packet-server-ip)
+          SSHOPTS="-o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /tmp/cluster/ssh-privatekey"
+          scp $SSHOPTS /usr/bin/openshift-tests /usr/bin/kubectl root@$IP:/usr/local/bin
+          oc config  set-cluster ostest --server=https://api.ostest.test.metalkube.org:6443 --insecure-skip-tls-verify
         fi
 
         mkdir -p /tmp/output
@@ -385,6 +402,16 @@ objects:
             --provider "${TEST_PROVIDER:-}" -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit
         }
 
+        function run-remote-tests() {
+          set +e
+          ssh $SSHOPTS root@$IP openshift-tests run "${TEST_SUITE}" \
+            --provider "'${TEST_PROVIDER:-}'" -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit
+          rv=$?
+          ssh $SSHOPTS root@$IP tar -czf - /tmp/artifacts | tar -C / -xzf - 
+          set -e
+          return $rv
+        }
+
         if [[ "${CLUSTER_TYPE}" == "gcp" ]]; then
           setup-google-cloud-sdk
         fi
@@ -450,6 +477,11 @@ objects:
         #!/bin/sh
         set -e
 
+        # setup/teardown is different for servers on packet.net (see setup-packet)
+        if [ "${CLUSTER_TYPE}" == "metal" ] ; then
+            exit 0
+        fi
+
         trap 'rc=$?; if test "${rc}" -eq 0; then touch /tmp/setup-success; else touch /tmp/exit; fi; exit "${rc}"' EXIT
         trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM
 
@@ -786,6 +818,12 @@ objects:
       - -c
       - |
         #!/bin/bash
+
+        # setup/teardown is different for servers on packet.net (see setup-packet)
+        if [ "${CLUSTER_TYPE}" == "metal" ] ; then
+            exit 0
+        fi
+
         function queue() {
           local TARGET="${1}"
           shift
@@ -932,3 +970,150 @@ objects:
           fi
           sleep 60 & wait
         done
+
+    # The setup-packet and test containers need libnns_wrapper to use ssh
+    # TODO(derekh): investigate if it can be added to that container images
+    - name: nss-wrapper-hack
+      image: ${IMAGE_LIBVIRT_INSTALLER}
+      terminationMessagePolicy: FallbackToLogsOnError
+      volumeMounts:
+      - name: shared-tmp
+        mountPath: /tmp/shared
+      command:
+      - /bin/bash
+      - -c
+      - |
+        #!/bin/bash
+        set -xe
+        cp /bin/mock-nss.sh /usr/lib64/libnss_wrapper.so /tmp/shared/
+    # We need to have a seperate setup container for packet.net servers
+    # as we need an image with terrafrom
+    - name: setup-packet
+      image: ${IMAGE_UPI_INSTALLER}
+      terminationMessagePolicy: FallbackToLogsOnError
+      volumeMounts:
+      - name: shared-tmp
+        mountPath: /tmp/shared
+      - name: cluster-profile
+        mountPath: /tmp/cluster
+      - name: artifacts
+        mountPath: /tmp/artifacts
+      env:
+      - name: CLUSTER_NAME
+        value: ${NAMESPACE}-${JOB_NAME_HASH}
+      - name: PACKET_PROJECT_ID
+        value: b3c1623c-ce0b-45cf-9757-c61a71e06eac
+      - name: PULL_SECRET_PATH
+        value: /tmp/cluster/pull-secret
+      - name: MIRROR_BASE
+        value: registry.svc.ci.openshift.org/${NAMESPACE}/release
+      command:
+      - /bin/sh
+      - -c
+      - |
+        #!/bin/sh
+        set -ex
+
+        if [ "${CLUSTER_TYPE}" != "metal" ] ; then
+            exit 0
+        fi
+
+        finished()
+        {
+            set +e
+
+            if [ -n "$IP" ] ; then
+                echo "Getting logs"
+                ssh $SSHOPTS root@$IP tar -czf - /root/dev-scripts/logs | tar -C /tmp/artifacts -xzf -
+                sed -i -e 's/.*auths.*/*** PULL_SECRET ***/g' /tmp/artifacts/root/dev-scripts/logs/*
+            fi
+
+            echo "Deprovisioning cluster ..."
+            cd /tmp/artifacts/terraform
+            terraform init
+            for r in {1..5}; do terraform destroy -auto-approve && break ; done
+            touch /tmp/shared/exit
+        }
+        trap finished EXIT TERM
+
+        mkdir -p /tmp/artifacts/terraform
+        cd /tmp/artifacts/terraform
+
+        set +x
+        export PACKET_AUTH_TOKEN=$(cat /tmp/cluster/.packetcred)
+        set -x
+
+        cat > /tmp/artifacts/terraform/terraform.tf <<-EOF
+        provider "packet" {
+        }
+
+        resource "packet_device" "server" {
+          count            = "1"
+          project_id       = "$PACKET_PROJECT_ID"
+          hostname         = "ipi-$CLUSTER_NAME"
+          plan             = "c2.medium.x86"
+          facilities       = ["ewr1", "ewr1", "sjc1"]
+          operating_system = "centos_7"
+          billing_cycle    = "hourly"
+        }
+
+        EOF
+
+        terraform init
+        # Packet returns transients errors when creating devices.
+        # example, `Oh snap, something went wrong! We've logged the error and will take a look - please reach out to us if you continue having trouble.`
+        # therefore the terraform apply needs to be retried a few time before giving up.
+        rc=1
+        for r in {1..5}; do terraform apply -auto-approve && rc=0 && break ; done
+        if test "${rc}" -eq 1; then echo "failed to create the infra resources"; sleep 1; fi
+
+        jq -r '.modules[0].resources["packet_device.server"].primary.attributes.access_public_ipv4' terraform.tfstate > /tmp/shared/packet-server-ip
+
+        export HOME=/tmp/shared/nss_wrapper
+        export NSS_WRAPPER_PASSWD=$HOME/passwd NSS_WRAPPER_GROUP=$HOME/group NSS_USERNAME=nsswrapper NSS_GROUPNAME=nsswrapper LD_PRELOAD=/tmp/shared/libnss_wrapper.so
+        mkdir -p $HOME
+        bash /tmp/shared/mock-nss.sh
+
+        SSHOPTS="-o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /tmp/cluster/ssh-privatekey"
+        export IP=$(cat /tmp/shared/packet-server-ip)
+
+        for x in $(seq 10) ; do
+            test $x == 10 && exit 1
+            ssh $SSHOPTS root@$IP hostname && break
+            sleep 10
+        done
+
+        scp $SSHOPTS ${PULL_SECRET_PATH} root@$IP:pull-secret
+        timeout -s 9 175m ssh $SSHOPTS root@$IP bash - << EOF |& sed -e 's/.*auths.*/*** PULL_SECRET ***/g'
+        set -ex
+
+        yum install -y git
+
+        # python2-cryptography needs to come from delorean-master-testing, priority of packet.repo overrides it
+        # remove the priority and instead ensure the packet repo is named first alphabetically
+        # this way it is prefered but it isn't a hard override when newer versions are found elsewhere
+        sed -i -e 's/priority.*//g' /etc/yum.repos.d/packet.repo
+        sed -i -e 's/packet-/a_packet-/g' /etc/yum.repos.d/packet.repo
+
+        mkdir -p /tmp/artifacts
+
+        git clone https://github.com/openshift-metal3/dev-scripts.git
+        cd dev-scripts
+
+        set +x
+        echo "export PULL_SECRET='\$(cat /root/pull-secret)'" > /root/dev-scripts/config_root.sh
+        set -x
+        # TODO(derekh): this needs to be commented back once ci images work
+        echo "export OPENSHIFT_RELEASE_IMAGE=$MIRROR_BASE:latest" >> /root/dev-scripts/config_root.sh
+        echo "export ADDN_DNS=\$(awk '/nameserver/ { print \$2;exit; }' /etc/resolv.conf)" >> /root/dev-scripts/config_root.sh
+        echo 'export KUBECONFIG=/root/dev-scripts/ocp/auth/kubeconfig' >> /root/.bashrc
+
+        timeout -s 9 105m make
+
+        EOF
+
+        mkdir -p /tmp/artifacts/installer/auth
+        scp $SSHOPTS root@$IP:./dev-scripts/ocp/auth/kubeconfig /tmp/artifacts/installer/auth/kubeconfig
+
+        touch /tmp/shared/setup-success
+        while [ ! -f /tmp/shared/exit ] ; do sleep 1 ; done