diff --git a/Makefile b/Makefile index 8dfb84a8d3359..8a15b94f693f4 100644 --- a/Makefile +++ b/Makefile @@ -138,6 +138,7 @@ prow-cluster-jobs: oc create configmap cluster-profile-gcp-ha --from-file=cluster/test-deploy/gcp/vars.yaml --from-file=cluster/test-deploy/gcp/vars-origin.yaml -o yaml --dry-run | oc apply -f - oc create configmap cluster-profile-gcp-ha-static --from-file=cluster/test-deploy/gcp/vars.yaml --from-file=cluster/test-deploy/gcp/vars-origin.yaml -o yaml --dry-run | oc apply -f - oc create configmap prow-job-cluster-launch-e2e --from-file=cluster/ci/config/prow/jobs/cluster-launch-e2e.yaml -o yaml --dry-run | oc apply -f - + oc create configmap prow-job-master-sidecar --from-file=cluster/ci/config/prow/jobs/master-sidecar.yaml -o yaml --dry-run | oc apply -f - .PHONY: prow-cluster-jobs prow-jobs: prow-cluster-jobs diff --git a/cluster/ci/config/prow/config.yaml b/cluster/ci/config/prow/config.yaml index 098116cee291b..8143141b4bc90 100644 --- a/cluster/ci/config/prow/config.yaml +++ b/cluster/ci/config/prow/config.yaml @@ -1012,10 +1012,10 @@ presubmits: agent: jenkins labels: master: ci.openshift.redhat.com - always_run: true + always_run: false context: ci/openshift-jenkins/integration - rerun_command: "/test integration" - trigger: "((?m)^/test( all| integration),?(\\s+|$))" + rerun_command: "/test old-integration" + trigger: "((?m)^/test( old-integration),?(\\s+|$))" - name: test_pull_request_image_registry_unit agent: jenkins labels: @@ -1031,7 +1031,7 @@ presubmits: - master rerun_command: "/test unit" always_run: true - trigger: "((?m)^/test( unit),?(\\s+|$))" + trigger: "((?m)^/test( all| unit),?(\\s+|$))" decorate: true spec: serviceAccountName: ci-operator @@ -1051,6 +1051,44 @@ presubmits: - --artifact-dir=$(ARTIFACTS) - --dry-run=false - --target=unit + - name: pull-ci-image-registry-integration + agent: kubernetes + context: ci/prow/integration + branches: + - master + rerun_command: "/test integration" + always_run: true + trigger: "((?m)^/test( all| integration),?(\\s+|$))" + decorate: true + spec: + serviceAccountName: ci-operator + volumes: + - name: job-definition + configMap: + name: prow-job-master-sidecar + containers: + - name: test + image: ci-operator:latest + volumeMounts: + - name: job-definition + mountPath: /usr/local/integration + subPath: master-sidecar.yaml + env: + - name: COMMAND + value: TEST_KUBECONFIG=${KUBECONFIG} ARTIFACT_DIR=/tmp/artifacts JUNIT_REPORT=true make test-integration + - name: CONFIG_SPEC + valueFrom: + configMapKeyRef: + name: ci-operator-image-registry + key: config.json + command: + - ci-operator + args: + - --delete-when-idle=10m + - --artifact-dir=$(ARTIFACTS) + - --dry-run=false + - --template=/usr/local/integration + - --target=integration - name: pull-ci-image-registry-e2e agent: kubernetes context: ci/prow/e2e @@ -1058,7 +1096,7 @@ presubmits: - master rerun_command: "/test e2e" always_run: true - trigger: "((?m)^/test( e2e),?(\\s+|$))" + trigger: "((?m)^/test( all| e2e),?(\\s+|$))" decorate: true spec: serviceAccountName: ci-operator diff --git a/cluster/ci/config/prow/jobs/cluster-launch-e2e.yaml b/cluster/ci/config/prow/jobs/cluster-launch-e2e.yaml index a606d730477cb..3036ef58a563a 100644 --- a/cluster/ci/config/prow/jobs/cluster-launch-e2e.yaml +++ b/cluster/ci/config/prow/jobs/cluster-launch-e2e.yaml @@ -24,6 +24,7 @@ parameters: - name: TEST_SUITE_SERIAL - name: TEST_FOCUS_SERIAL - name: TEST_SKIP_SERIAL + objects: # We want the cluster to be able to access these images diff --git a/cluster/ci/config/prow/jobs/master-sidecar.yaml b/cluster/ci/config/prow/jobs/master-sidecar.yaml new file mode 100644 index 0000000000000..f2f5444c6147d --- /dev/null +++ b/cluster/ci/config/prow/jobs/master-sidecar.yaml @@ -0,0 +1,119 @@ +kind: Template +apiVersion: template.openshift.io/v1 + +parameters: +- name: JOB_NAME_SAFE + required: true +- name: IMAGE_CONTROL_PLANE + required: true +- name: LOCAL_IMAGE_TEST_BIN + required: true +- name: COMMAND + required: true +- name: NAMESPACE + required: true + +objects: + +# We want the cluster to be able to access these images +- kind: RoleBinding + apiVersion: authorization.openshift.io/v1 + metadata: + name: ${JOB_NAME_SAFE}-image-puller + namespace: ${NAMESPACE} + roleRef: + name: system:image-puller + subjects: + - kind: SystemGroup + name: system:unauthenticated + +# The pod spins up a simple openshift control plane as a sidecar and waits for the +# COMMAND specified to the template to be executed, before itself exiting. The test +# container is given access to the generated config and the admin.kubeconfig. +- kind: Pod + apiVersion: v1 + metadata: + name: ${JOB_NAME_SAFE} + namespace: ${NAMESPACE} + spec: + restartPolicy: Never + activeDeadlineSeconds: 7200 + terminationGracePeriodSeconds: 600 + volumes: + - name: artifacts + emptyDir: {} + - name: shared-tmp + emptyDir: {} + + containers: + + # Once admin.kubeconfig exists, executes shared tests + - name: test + image: ${LOCAL_IMAGE_TEST_BIN} + volumeMounts: + - name: shared-tmp + mountPath: /tmp/shared + - name: artifacts + mountPath: /tmp/artifacts + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -euo pipefail + + trap 'touch /tmp/shared/exit' EXIT + trap 'kill $(jobs -p); exit 0' TERM + + # wait until the master job creates admin.kubeconfig + while true; do + if [[ ! -f /tmp/shared/admin.kubeconfig ]]; then + sleep 5 & wait + continue + fi + break + done + echo "Found kubeconfig" + export KUBECONFIG=/tmp/shared/admin.kubeconfig + + ${COMMAND} + + # Start a standalone master + - name: control-plane + image: ${IMAGE_CONTROL_PLANE} + volumeMounts: + - name: shared-tmp + mountPath: /tmp/shared + - name: artifacts + mountPath: /tmp/artifacts + env: + - name: TYPE + value: ${CLUSTER_TYPE} + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -euo pipefail + function teardown() { + set +e + echo "Gathering artifacts ..." + } + trap 'teardown' EXIT + trap 'kill $(jobs -p); exit 0' TERM + + ( + while true; do + if [[ -f /tmp/shared/exit ]]; then + echo "Shutting down" + kill 1 + exit 0 + fi + sleep 5 + done + ) & + + mkdir /tmp/cluster && cd /tmp/cluster + openshift start master --write-config=/tmp/shared + openshift start master --config=/tmp/shared/master-config.yaml & + wait $!