diff --git a/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml b/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml index 529b5cd31f10c..09d0e9b423af2 100644 --- a/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml +++ b/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml @@ -1,3 +1,12 @@ +base_images: + cli-operator-sdk: + name: cli-operator-sdk + namespace: ocp + tag: v1.39.2 + dev-scripts: + name: test + namespace: ocp-kni + tag: dev-scripts binary_build_commands: make build build_root: image_stream_tag: @@ -9,7 +18,8 @@ images: items: - dockerfile_path: Dockerfile.openshift to: openperouter-operator - - dockerfile_path: operator/bundle.Dockerfile.openshift + - context_dir: operator + dockerfile_path: bundle.Dockerfile to: openperouter-operator-bundle - build_args: - name: BASE_IMAGE @@ -19,10 +29,11 @@ images: operator: bundles: - as: operator-bundle - dockerfile_path: operator/bundle.Dockerfile.openshift + context_dir: operator + dockerfile_path: bundle.Dockerfile skip_building_index: true substitutions: - - pullspec: quay.io/redhat-user-workloads/telco-5g-tenant/openperouter-operator-[\d].* + - pullspec: quay.io/openperouter/router:main with: pipeline:openperouter-operator promotion: to: @@ -47,6 +58,27 @@ resources: requests: cpu: 100m memory: 200Mi +tests: +- as: openperouter-e2e-metal + cluster: build05 + steps: + cluster_profile: equinix-ocp-metal + dependencies: + OO_BUNDLE: operator-bundle + env: + DEVSCRIPTS_CONFIG: | + IP_STACK=v4v6 + NETWORK_TYPE=OVNKubernetes + EXTRA_NETWORK_NAMES="toswitch1 toswitch2" + TOSWITCH1_NETWORK_SUBNET_V4='192.168.11.0/24' + TOSWITCH1_NETWORK_SUBNET_V6='2001:db8:11::/64' + TOSWITCH2_NETWORK_SUBNET_V4='192.168.12.0/24' + TOSWITCH2_NETWORK_SUBNET_V6='2001:db8:12::/64' + NUM_WORKERS=2 + ENABLE_LOCAL_REGISTRY=true + OO_INSTALL_MODE: AllNamespaces + OO_INSTALL_NAMESPACE: openshift-openperouter-system + workflow: baremetalds-openperouter-e2e zz_generated_metadata: branch: main org: openshift-kni diff --git a/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml b/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml index e7cd5b88239fb..c3f5699420109 100644 --- a/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml @@ -111,3 +111,86 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )images,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build05 + context: ci/prow/openperouter-e2e-metal + decorate: true + decoration_config: {} + labels: + ci-operator.openshift.io/cloud: equinix-ocp-metal + ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal + ci-operator.openshift.io/cluster: build05 + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-kni-openperouter-main-openperouter-e2e-metal + path_alias: github.com/openperouter/openperouter + rerun_command: /test openperouter-e2e-metal + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=openperouter-e2e-metal + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )openperouter-e2e-metal,?($|\s.*) diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS b/ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS new file mode 100644 index 0000000000000..d90ed11dd94d7 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS @@ -0,0 +1,10 @@ +approvers: +- fedepaol +- maiqueb +- oribon +- zeeke +reviewers: +- fedepaol +- maiqueb +- oribon +- zeeke \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json new file mode 100644 index 0000000000000..bcac85f36d446 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml", + "owners": { + "approvers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ], + "reviewers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml new file mode 100644 index 0000000000000..1a09cf7f1fed2 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml @@ -0,0 +1,22 @@ +workflow: + as: baremetalds-openperouter-e2e + steps: + pre: + - ref: ofcir-acquire + - ref: ipi-install-rbac + - ref: baremetalds-devscripts-ibm + - ref: baremetalds-devscripts-proxy + - ref: ipi-install-hosted-loki + - ref: rhcos-conf-osstream + - ref: baremetalds-openperouter-e2e-iptables + - ref: baremetalds-devscripts-setup + test: + - ref: optional-operators-operator-sdk + - ref: baremetalds-openperouter-e2e-test + post: + - chain: baremetalds-ofcir-post + documentation: |- + Deploy OpenPerOuter on a baremetal OCP cluster and verify all components + come up healthy. Uses firewalld REJECT (tcp-reset) rules on the dev-scripts + host to keep the bootstrap gather from stalling on routed toswitch IPv6 + addresses. diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS new file mode 100644 index 0000000000000..d90ed11dd94d7 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS @@ -0,0 +1,10 @@ +approvers: +- fedepaol +- maiqueb +- oribon +- zeeke +reviewers: +- fedepaol +- maiqueb +- oribon +- zeeke \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh new file mode 100644 index 0000000000000..b394e16a560fd --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +echo "************ openperouter firewalld fix for bootstrap gather ************" + +# shellcheck source=/dev/null +source "${SHARED_DIR}/packet-conf.sh" + +# The installer's bootstrap gather runs on the bootstrap node and SSHes to every +# NIC address of every cluster node. The extra-network (toswitch1/toswitch2) +# addresses are unreachable, and the IPv6 ones in particular stall for ~7min each +# (TCP half-opens, then the SSH key exchange is reset), adding well over an hour +# to the gather. bootstrap->toswitch traffic is *routed* between libvirt bridges +# through the dev-scripts host, so it traverses this host's FORWARD chain. We add +# REJECT --reject-with tcp-reset rules there so those SYNs get an immediate RST +# instead of stalling, turning each ~7min stall into an instant failure. +# +# Notes: +# - Rules go in FORWARD (routed node-to-node gather traffic) and OUTPUT +# (host-originated `openshift-install gather bootstrap` legs). +# - We use firewalld --permanent rules, NOT raw `iptables -I`. dev-scripts runs +# `firewall-cmd --reload` during setup, which flushes manually-inserted rules; +# permanent firewalld rules survive the reload. +# - Only tcp/22 to the toswitch subnets is rejected; the EVPN/SRv6 fabric uses +# BGP/BFD/VXLAN, not SSH, so the fabric E2E is unaffected. +# - Link-local (fe80::) gather attempts are on-link and never routed through this +# host, so they cannot be intercepted here. + +# Parse subnet vars from DEVSCRIPTS_CONFIG +eval "${DEVSCRIPTS_CONFIG}" + +ssh "${SSHOPTS[@]}" "root@${IP}" bash -s -- \ + "${TOSWITCH1_NETWORK_SUBNET_V4}" "${TOSWITCH2_NETWORK_SUBNET_V4}" \ + "${TOSWITCH1_NETWORK_SUBNET_V6}" "${TOSWITCH2_NETWORK_SUBNET_V6}" << 'EOFFIREWALL' +set -euo pipefail + +# dev-scripts depends on firewalld; make sure it is running before we add rules. +systemctl is-active --quiet firewalld || systemctl start firewalld + +# add_reject : reject tcp/22 to on FORWARD + OUTPUT +add_reject() { + local family="$1" subnet="$2" chain + for chain in FORWARD OUTPUT; do + firewall-cmd --permanent --direct --add-rule "${family}" filter "${chain}" 0 \ + -p tcp -d "${subnet}" --dport 22 -j REJECT --reject-with tcp-reset + done +} + +add_reject ipv4 "$1" +add_reject ipv4 "$2" +add_reject ipv6 "$3" +add_reject ipv6 "$4" + +# Apply permanent -> runtime. +firewall-cmd --reload + +echo "Added firewalld REJECT(tcp-reset) rules for toswitch subnets on FORWARD+OUTPUT" +firewall-cmd --direct --get-all-rules +EOFFIREWALL diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json new file mode 100644 index 0000000000000..82f34e850ad9a --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml", + "owners": { + "approvers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ], + "reviewers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml new file mode 100644 index 0000000000000..c5d27d255720d --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml @@ -0,0 +1,21 @@ +ref: + as: baremetalds-openperouter-e2e-iptables + from: dev-scripts + commands: baremetalds-openperouter-e2e-iptables-commands.sh + timeout: 300s + env: + - name: DEVSCRIPTS_CONFIG + default: "" + documentation: Dev-scripts config containing TOSWITCH*_NETWORK_SUBNET_V4/V6 vars + resources: + requests: + cpu: 100m + memory: 200Mi + documentation: |- + Add firewalld REJECT (tcp-reset) rules on the dev-scripts host FORWARD and + OUTPUT chains for tcp/22 to the toswitch1/toswitch2 subnets. The installer's + bootstrap gather SSHes to every node NIC address; the routed, unreachable + toswitch IPv6 addresses otherwise stall the gather ~7min each. An immediate + RST turns each stall into an instant failure. Uses permanent firewalld rules + so they survive the `firewall-cmd --reload` that dev-scripts runs during + setup. Link-local (fe80::) attempts are on-link and cannot be handled here. diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS new file mode 100644 index 0000000000000..9827a2117fb39 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS @@ -0,0 +1,11 @@ +approvers: +- fedepaol +- maiqueb +- oribon +- zeeke +reviewers: +- fedepaol +- maiqueb +- oribon +- zeeke + diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh new file mode 100644 index 0000000000000..e1e3edce4083c --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +echo "************ openperouter deploy-verify test ************" + +# shellcheck source=/dev/null +source "${SHARED_DIR}/packet-conf.sh" + +echo "### Copying openperouter PR source to remote host" +OPENPEROUTER_SRC="/go/src/github.com/openperouter/openperouter" +ssh "${SSHOPTS[@]}" "root@${IP}" "mkdir -p /root/openperouter" +scp "${SSHOPTS[@]}" "${OPENPEROUTER_SRC}/Makefile" "root@${IP}:/root/openperouter/" +scp "${SSHOPTS[@]}" -r "${OPENPEROUTER_SRC}/e2etests" "root@${IP}:/root/openperouter/" + +echo "### Create OpenPERouter CR and verify deployment" +ssh "${SSHOPTS[@]}" "root@${IP}" bash -s << 'EOFDEPLOY' +set -euo pipefail +export KUBECONFIG=/root/dev-scripts/ocp/ostest/auth/kubeconfig + +# Ensure namespace is privileged (router pods need host networking + nsenter) +oc label --overwrite ns openshift-openperouter-system \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/audit=privileged \ + pod-security.kubernetes.io/warn=privileged + +# Create OpenPERouter CR +cat <<'EOF' | oc apply -f - +apiVersion: network.openperouter.io/v1alpha1 +kind: OpenPERouter +metadata: + name: openperouter + namespace: openshift-openperouter-system +spec: + logLevel: debug +EOF + +# Wait for controller and router daemonsets to be created and rolled out +for ds in controller router; do + echo "Waiting for daemonset $ds to be created..." + deadline=$((SECONDS + 300)) + until oc get daemonset "$ds" -n openshift-openperouter-system &>/dev/null; do + if (( SECONDS >= deadline )); then + echo "ERROR: Timed out waiting for daemonset $ds" + exit 1 + fi + sleep 5 + done + oc rollout status daemonset/"$ds" -n openshift-openperouter-system --timeout=300s +done + +echo "=== Deploy verification ===" +oc get pods -n openshift-openperouter-system -o wide +oc get daemonset -n openshift-openperouter-system + +# Verify all pods are Running and Ready +NOT_READY=$(oc get pods -n openshift-openperouter-system --no-headers | grep -v "Completed" | grep -v "1/1\|2/2\|3/3\|4/4\|5/5" || true) +if [ -n "$NOT_READY" ]; then + echo "ERROR: Some pods are not fully ready:" + echo "$NOT_READY" + exit 1 +fi + +echo "All openperouter pods are running and ready" +EOFDEPLOY diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json new file mode 100644 index 0000000000000..c17b6fc2b2caa --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml", + "owners": { + "approvers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ], + "reviewers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml new file mode 100644 index 0000000000000..8b41d93b8becf --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml @@ -0,0 +1,14 @@ +ref: + as: baremetalds-openperouter-e2e-test + from: src + grace_period: 10m + commands: baremetalds-openperouter-e2e-test-commands.sh + timeout: 3600s + resources: + requests: + cpu: 100m + memory: 200Mi + documentation: |- + Create OpenPERouter CR and verify that operator, controller, and router + daemonsets come up healthy. OLM bundle deploy is handled by the + optional-operators-operator-sdk step.