openshift · qinqon · Nov 28, 2023
diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.16__periodics.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.16__periodics.yaml
@@ -144,6 +144,15 @@ tests:
     env:
       CNV_PRERELEASE_VERSION: "4.16"
     workflow: hypershift-kubevirt-csi-e2e
+- as: e2e-kubevirt-baremetalds-secondary-network-conformance
+  cron: 0 4 * * *
+  steps:
+    cluster_profile: equinix-ocp-metal
+    env:
+      HYPERSHIFT_NODE_COUNT: "2"
+      ODF_OPERATOR_SUB_CHANNEL: stable-4.14
+      USE_SECONDARY_NETWORK: "true"
+    workflow: hypershift-kubevirt-baremetalds-conformance
 zz_generated_metadata:
   branch: release-4.16
   org: openshift

diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-4.16-periodics.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-4.16-periodics.yaml
@@ -611,6 +611,87 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build01
+  cron: 0 4 * * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.16
+    org: openshift
+    repo: hypershift
+  labels:
+    ci-operator.openshift.io/cloud: equinix-ocp-metal
+    ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal
+    ci-operator.openshift.io/variant: periodics
+    ci.openshift.io/generator: prowgen
+    job-release: "4.16"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-hypershift-release-4.16-periodics-e2e-kubevirt-baremetalds-secondary-network-conformance
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --secret-dir=/usr/local/e2e-kubevirt-baremetalds-secondary-network-conformance-cluster-profile
+      - --target=e2e-kubevirt-baremetalds-secondary-network-conformance
+      - --variant=periodics
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /usr/local/e2e-kubevirt-baremetalds-secondary-network-conformance-cluster-profile
+        name: cluster-profile
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: cluster-profile
+      secret:
+        secretName: cluster-secrets-equinix-ocp-metal
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build01
   cron: 0 4 * * *

diff --git a/...ubevirt/baremetalds/conformance/hypershift-kubevirt-baremetalds-conformance-workflow.yaml b/...ubevirt/baremetalds/conformance/hypershift-kubevirt-baremetalds-conformance-workflow.yaml
@@ -28,15 +28,17 @@ workflow:
     pre:
     - chain: baremetalds-ipi-pre
     - ref: enable-qe-catalogsource
+    - chain: hypershift-kubevirt-baremetalds-kubernetes-nmstate
     - chain: hypershift-kubevirt-baremetalds-metallb
-    - chain: hypershift-kubevirt-baremetalds-odf
     - ref: hypershift-kubevirt-install
     - ref: hypershift-install
+    - chain: hypershift-kubevirt-baremetalds-odf
     - chain: hypershift-kubevirt-create
     - chain: hypershift-kubevirt-baremetalds-proxy
     env:
       METALLB_OPERATOR_SUB_SOURCE: qe-app-registry
       LOCAL_STORAGE_OPERATOR_SUB_SOURCE: qe-app-registry
+      KUBERNETES_NMSTATE_OPERATOR_SUB_SOURCE: qe-app-registry
       ODF_OPERATOR_SUB_SOURCE: redhat-operators-v4-14
       TEST_SKIPS: StatefulSet Basic \| StatefulSet Non-retain
       DEVSCRIPTS_CONFIG: |
@@ -49,3 +51,9 @@ workflow:
         VM_EXTRADISKS=true
         VM_EXTRADISKS_LIST=vda
         VM_EXTRADISKS_SIZE=100G
+        INSTALLER_PROXY=true
+        EXTRA_NETWORK_NAMES="net1 net2"
+        NET1_NETWORK_SUBNET_V4="192.168.221.0/24"
+        NET1_NETWORK_SUBNET_V6="fd2e:6f44:5dd8:ca56::/120"
+        NET2_NETWORK_SUBNET_V4="192.168.222.0/24"
+        NET2_NETWORK_SUBNET_V6="fd2e:6f44:5dd8:cc56::/120"
diff --git a/ci-operator/step-registry/hypershift/kubevirt/baremetalds/kubernetes-nmstate/OWNERS b/ci-operator/step-registry/hypershift/kubevirt/baremetalds/kubernetes-nmstate/OWNERS
@@ -0,0 +1,15 @@
+approvers:
+- davidvossel
+- nirarg
+- nunnatsa
+- qinqon
+- orenc1
+- LiangquanLi930
+options: {}
+reviewers:
+- davidvossel
+- nirarg
+- nunnatsa
+- qinqon
+- orenc1
+- LiangquanLi930
diff --git a/...kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-chain.metadata.json b/...kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-chain.metadata.json
@@ -0,0 +1,21 @@
+{
+	"path": "hypershift/kubevirt/baremetalds/kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-chain.yaml",
+	"owners": {
+		"approvers": [
+			"davidvossel",
+			"nirarg",
+			"nunnatsa",
+			"qinqon",
+			"orenc1",
+			"LiangquanLi930"
+		],
+		"reviewers": [
+			"davidvossel",
+			"nirarg",
+			"nunnatsa",
+			"qinqon",
+			"orenc1",
+			"LiangquanLi930"
+		]
+	}
+}
diff --git a/...emetalds/kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-chain.yaml b/...emetalds/kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-chain.yaml
@@ -0,0 +1,7 @@
+chain:
+  as: hypershift-kubevirt-baremetalds-kubernetes-nmstate
+  steps:
+  - ref: operatorhub-subscribe-kubernetes-nmstate-operator
+  - ref: hypershift-kubevirt-baremetalds-kubernetes-nmstate
+  documentation: |-
+    This step to setup NMState CR for Kubernetes in OpenShift BM
diff --git a/...metalds/kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-commands.sh b/...metalds/kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-commands.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+set -x
+
+HANDLER_NAMESPACE=openshift-nmstate
+
+cat << EOF | oc apply -f -
+apiVersion: nmstate.io/v1
+kind: NMState
+metadata:
+  name: nmstate
+EOF
+
+# Wait a little for resources to be created
+sleep 5
+
+oc rollout status -w -n ${HANDLER_NAMESPACE} ds nmstate-handler --timeout=2m
+oc rollout status -w -n ${HANDLER_NAMESPACE} deployment nmstate-webhook --timeout=2m
diff --git a/...s/kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-ref.metadata.json b/...s/kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-ref.metadata.json
@@ -0,0 +1,21 @@
+{
+	"path": "hypershift/kubevirt/baremetalds/kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-ref.yaml",
+	"owners": {
+		"approvers": [
+			"davidvossel",
+			"nirarg",
+			"nunnatsa",
+			"qinqon",
+			"orenc1",
+			"LiangquanLi930"
+		],
+		"reviewers": [
+			"davidvossel",
+			"nirarg",
+			"nunnatsa",
+			"qinqon",
+			"orenc1",
+			"LiangquanLi930"
+		]
+	}
+}
diff --git a/...aremetalds/kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-ref.yaml b/...aremetalds/kubernetes-nmstate/hypershift-kubevirt-baremetalds-kubernetes-nmstate-ref.yaml
@@ -0,0 +1,12 @@
+ref:
+  as: hypershift-kubevirt-baremetalds-kubernetes-nmstate
+  from: cli
+  grace_period: 10m
+  commands: hypershift-kubevirt-baremetalds-kubernetes-nmstate-commands.sh
+  resources:
+    requests:
+      cpu: 1000m
+      memory: 500Mi
+  timeout: 10m0s
+  documentation: |-
+    This step to setup Kubernetes NMState for Kubernetes in OpenShift BM, create NMState
diff --git a/ci-operator/step-registry/hypershift/kubevirt/create/hypershift-kubevirt-create-chain.yaml b/ci-operator/step-registry/hypershift/kubevirt/create/hypershift-kubevirt-create-chain.yaml
@@ -27,6 +27,9 @@ chain:
       default: ""
       documentation: |-
         The storage class that will be used for etcd on the hosted control plane.
+    - name: USE_SECONDARY_NETWORK
+      default: "false"
+      documentation: "Use non default networks"
     commands: |-
       set -exuo pipefail
 
@@ -63,6 +66,87 @@ chain:
 
       CLUSTER_NAME="$(echo -n $PROW_JOB_ID|sha256sum|cut -c-20)"
       CLUSTER_NAMESPACE=clusters-${CLUSTER_NAME}
+
+      if [ "${USE_SECONDARY_NETWORK}" == "true" ]; then
+        oc apply -f - <<EOF
+      apiVersion: nmstate.io/v1
+      kind: NodeNetworkConfigurationPolicy
+      metadata:
+        name: multi-net
+      spec:
+        desiredState:
+          interfaces:
+          - name: net1
+            type:  linux-bridge
+            state: up
+            ipv4:
+              enabled: false
+            ipv6:
+              enabled: false
+            bridge:
+              options:
+                stp:
+                  enabled: true
+              port:
+                - name: enp3s0
+          - name: net2
+            type:  linux-bridge
+            state: up
+            ipv4:
+              enabled: false
+            ipv6:
+              enabled: false
+            bridge:
+              options:
+                stp:
+                  enabled: true
+              port:
+                - name: enp4s0
+      EOF
+        oc wait nncp multi-net --for condition=Available --timeout 5m
+        oc apply -f - <<EOF
+      ---
+      apiVersion: k8s.cni.cncf.io/v1
+      kind: NetworkAttachmentDefinition
+      metadata:
+        namespace: default
+        name: net1
+        annotations:
+          k8s.v1.cni.cncf.io/resourceName: bridge.network.kubevirt.io/net1
+      spec:
+        config: >
+          {
+              "cniVersion": "0.3.1",
+              "name": "net1",
+              "plugins": [{
+                  "type": "cnv-bridge",
+                  "bridge": "net1",
+                  "ipam": {}
+              }]
+          }
+      ---
+      apiVersion: k8s.cni.cncf.io/v1
+      kind: NetworkAttachmentDefinition
+      metadata:
+        namespace: default
+        name: net2
+        annotations:
+          k8s.v1.cni.cncf.io/resourceName: bridge.network.kubevirt.io/net2
+      spec:
+        config: >
+          {
+              "cniVersion": "0.3.1",
+              "name": "net2",
+              "plugins": [{
+                  "type": "cnv-bridge",
+                  "bridge": "net2",
+                  "ipam": {}
+              }]
+          }
+      ---
+      EOF
+        EXTRA_ARGS="${EXTRA_ARGS} --additional-network=name:default/net1 --additional-network=name:default/net2 --attach-default-network=false"
+      fi
 
       echo "$(date) Creating HyperShift guest cluster ${CLUSTER_NAME}"
       $HCP_CLI create cluster kubevirt \

diff --git a/...irt/baremetalds/conformance/hypershift-mce-kubevirt-baremetalds-conformance-workflow.yaml b/...irt/baremetalds/conformance/hypershift-mce-kubevirt-baremetalds-conformance-workflow.yaml
@@ -48,3 +48,8 @@ workflow:
         VM_EXTRADISKS=true
         VM_EXTRADISKS_LIST=vda
         VM_EXTRADISKS_SIZE=100G
+        EXTRA_NETWORK_NAMES="net1 net2"
+        NET1_NETWORK_SUBNET_V4="192.168.221.0/24"
+        NET1_NETWORK_SUBNET_V6="fd2e:6f44:5dd8:ca56::/120"
+        NET2_NETWORK_SUBNET_V4="192.168.222.0/24"
+        NET2_NETWORK_SUBNET_V6="fd2e:6f44:5dd8:cc56::/120"