openshift · qinqon · Mar 8, 2023 · Mar 8, 2023 · Mar 8, 2023 · Mar 10, 2023
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -387,6 +387,10 @@ jobs:
           - {"target": "compact-mode",      "ha": "noHA", "gateway-mode": "local",  "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-disabled"}
           - {"target": "multi-homing",      "ha": "noHA", "gateway-mode": "local",  "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}
           - {"target": "multi-node-zones",  "ha": "noHA", "gateway-mode": "local",  "ipfamily": "ipv4",      "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "ic": "ic-multi-node-zones", "num-workers": "3", "num-nodes-per-zone": "2"}
+          - {"target": "kv-live-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "ic": "ic-disabled"}
+          - {"target": "kv-live-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "ic": "ic-disabled"}
+          - {"target": "kv-live-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "ic": "ic-multi-node-zones", "num-workers": "3", "num-nodes-per-zone": "2"}
+          - {"target": "kv-live-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "ic":  "ic-multi-node-zones", "num-workers": "3", "num-nodes-per-zone": "2"}
     needs: [ build-pr ]
     env:
       JOB_NAME: "${{ matrix.target }}-${{ matrix.ha }}-${{ matrix.gateway-mode }}-${{ matrix.ipfamily }}-${{ matrix.disable-snat-multiple-gws }}-${{ matrix.second-bridge }}"
@@ -401,6 +405,7 @@ jobs:
       KIND_IPV4_SUPPORT: "${{ matrix.ipfamily == 'IPv4' || matrix.ipfamily == 'dualstack' }}"
       KIND_IPV6_SUPPORT: "${{ matrix.ipfamily == 'IPv6' || matrix.ipfamily == 'dualstack' }}"
       ENABLE_MULTI_NET: "${{ matrix.target == 'multi-homing' }}"
+      KIND_INSTALL_KUBEVIRT: "${{ matrix.target == 'kv-live-migration' }}"
       OVN_COMPACT_MODE: "${{ matrix.target == 'compact-mode' }}"
       OVN_DUMMY_GATEWAY_BRIDGE: "${{ matrix.target == 'compact-mode' }}"
       OVN_ENABLE_INTERCONNECT: "${{ matrix.ic == 'ic-single-node-zones' ||  matrix.ic == 'ic-multi-node-zones'}}"
@@ -464,6 +469,8 @@ jobs:
           SINGLE_NODE_CLUSTER="true" make -C test shard-network
         elif [ "${{ matrix.target }}" == "multi-node-zones" ]; then
           make -C test control-plane WHAT="Multi node zones interconnect"
+        elif [ "${{ matrix.target }}" == "kv-live-migration" ]; then
+          make -C test control-plane WHAT="Kubevirt Virtual Machines"
         else
           make -C test ${{ matrix.target }}
         fi

diff --git a/Dockerfile b/Dockerfile
@@ -27,6 +27,8 @@ USER root
 
 ENV PYTHONDONTWRITEBYTECODE yes
 
+COPY *.rpm /root/
+
 # more-pkgs file is updated in Dockerfile.base
 # more-pkgs file contains the following ovs/ovn packages to be installed in this Dockerfile
 # - openvswitch-devel
@@ -43,7 +45,7 @@ RUN INSTALL_PKGS=" \
 	" && \
 	dnf install -y --nodocs $INSTALL_PKGS && \
 	eval "dnf install -y --nodocs $(cat /more-pkgs)" && \
-	dnf clean all && rm -rf /var/cache/*
+	dnf clean all && rm -rf /root/*.rpm && rm -rf /var/cache/*
 
 COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_output/go/bin/ovnkube /usr/bin/
 COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_output/go/bin/ovn-kube-util /usr/bin/

diff --git a/Dockerfile.base b/Dockerfile.base
@@ -15,17 +15,19 @@ RUN dnf install -y --nodocs \
 ARG ovsver=3.1.0-10.el9fdp
 ARG ovnver=23.03.0-49.el9fdp
 
+COPY *.rpm /root/
+
 RUN INSTALL_PKGS="iptables" && \
 	ovsver_short=$(echo "$ovsver" | cut -d'.' -f1,2) && \
 	ovnver_short=$(echo "$ovnver" | cut -d'.' -f1,2) && \
 	dnf install -y --nodocs $INSTALL_PKGS && \
 	dnf install -y --nodocs "openvswitch$ovsver_short = $ovsver" "python3-openvswitch$ovsver_short = $ovsver" && \
-	dnf install -y --nodocs "ovn$ovnver_short = $ovnver" "ovn$ovnver_short-central = $ovnver" "ovn$ovnver_short-host = $ovnver" && \
-	dnf clean all && rm -rf /var/cache/*
+    dnf install -y /root/ovn23.06-23.06.0-18.el9fdp.x86_64.rpm /root/ovn23.06-central-23.06.0-18.el9fdp.x86_64.rpm /root/ovn23.06-host-23.06.0-18.el9fdp.x86_64.rpm && \
+	dnf clean all && rm /root/*.rpm && rm -rf /var/cache/*
 
 RUN ovsver_short=$(echo "$ovsver" | cut -d'.' -f1,2) && \
 	ovnver_short=$(echo "$ovnver" | cut -d'.' -f1,2) && \
-	sed 's/%/"/g' <<<"%openvswitch$ovsver_short-devel = $ovsver% %openvswitch$ovsver_short-ipsec = $ovsver% %ovn$ovnver_short-vtep = $ovnver%" > /more-pkgs
+	sed 's/%/"/g' <<<"%openvswitch$ovsver_short-devel = $ovsver% %openvswitch$ovsver_short-ipsec = $ovsver% /root/ovn23.06-vtep-23.06.0-18.el9fdp.x86_64.rpm" > /more-pkgs
 
 RUN mkdir -p /var/run/openvswitch && \
     mkdir -p /var/run/ovn && \

diff --git a/contrib/kind.sh b/contrib/kind.sh
@@ -189,6 +189,8 @@ parse_args() {
                                                 ;;
             -pl | --install-cni-plugins )       KIND_INSTALL_PLUGINS=true
                                                 ;;
+            -ikv | --install-kubevirt)          KIND_INSTALL_KUBEVIRT=true
+                                                ;;
             -ha | --ha-enabled )                OVN_HA=true
                                                 ;;
             -me | --multicast-enabled)          OVN_MULTICAST_ENABLE=true
@@ -357,6 +359,7 @@ print_params() {
      echo "KIND_INSTALL_INGRESS = $KIND_INSTALL_INGRESS"
      echo "KIND_INSTALL_METALLB = $KIND_INSTALL_METALLB"
      echo "KIND_INSTALL_PLUGINS = $KIND_INSTALL_PLUGINS"
+     echo "KIND_INSTALL_KUBEVIRT = $KIND_INSTALL_KUBEVIRT"
      echo "OVN_HA = $OVN_HA"
      echo "RUN_IN_CONTAINER = $RUN_IN_CONTAINER"
      echo "KIND_CLUSTER_NAME = $KIND_CLUSTER_NAME"
@@ -498,11 +501,12 @@ set_default_params() {
   fi
   RUN_IN_CONTAINER=${RUN_IN_CONTAINER:-false}
   KIND_IMAGE=${KIND_IMAGE:-kindest/node}
-  K8S_VERSION=${K8S_VERSION:-v1.26.0}
+  K8S_VERSION=${K8S_VERSION:-v1.26.3}
   OVN_GATEWAY_MODE=${OVN_GATEWAY_MODE:-shared}
   KIND_INSTALL_INGRESS=${KIND_INSTALL_INGRESS:-false}
   KIND_INSTALL_METALLB=${KIND_INSTALL_METALLB:-false}
   KIND_INSTALL_PLUGINS=${KIND_INSTALL_PLUGINS:-false}
+  KIND_INSTALL_KUBEVIRT=${KIND_INSTALL_KUBEVIRT:-false}
   OVN_HA=${OVN_HA:-false}
   KIND_LOCAL_REGISTRY=${KIND_LOCAL_REGISTRY:-false}
   KIND_LOCAL_REGISTRY_NAME=${KIND_LOCAL_REGISTRY_NAME:-kind-registry}
@@ -579,7 +583,7 @@ set_default_params() {
   OVN_HOST_NETWORK_NAMESPACE=${OVN_HOST_NETWORK_NAMESPACE:-ovn-host-network}
   OVN_EGRESSIP_HEALTHCHECK_PORT=${OVN_EGRESSIP_HEALTHCHECK_PORT:-9107}
   OCI_BIN=${KIND_EXPERIMENTAL_PROVIDER:-docker}
-  OVN_DEPLOY_PODS=${OVN_DEPLOY_PODS:-"ovnkube-master ovnkube-node"}
+  OVN_DEPLOY_PODS=${OVN_DEPLOY_PODS:-"ovnkube-zone-controller ovnkube-control-plane ovnkube-master ovnkube-node"}
   OVN_METRICS_SCALE_ENABLE=${OVN_METRICS_SCALE_ENABLE:-false}
   OVN_ISOLATED=${OVN_ISOLATED:-false}
   OVN_GATEWAY_OPTS=${OVN_GATEWAY_OPTS:-""}
@@ -748,7 +752,9 @@ docker_disable_ipv6() {
 
 coredns_patch() {
   dns_server="8.8.8.8"
-  if [ "$KIND_IPV6_SUPPORT" == true ]; then
+  # No need for ipv6 nameserver for dual stack, it will ask for 
+  # A and AAAA records
+  if [ "$IP_FAMILY" == "ipv6" ]; then
     dns_server="2001:4860:4860::8888"
   fi
 
@@ -805,6 +811,12 @@ build_ovn_image() {
       $OCI_BIN push "${OVN_IMAGE}"
     fi
     popd
+  # We should push to local registry if image is not remote
+  elif [ "${OVN_IMAGE}" != "" -a "${KIND_LOCAL_REGISTRY}" == true ] && (echo "$OVN_IMAGE" | grep / -vq); then 
+    local local_registry_ovn_image="localhost:5000/${OVN_IMAGE}"
+    $OCI_BIN tag "$OVN_IMAGE" $local_registry_ovn_image
+    OVN_IMAGE=$local_registry_ovn_image
+    $OCI_BIN push $OVN_IMAGE
   fi
 }
 
@@ -1226,6 +1238,46 @@ add_dns_hostnames() {
   done
 }
 
+function is_nested_virt_enabled() {
+    local kvm_nested="unknown"
+    if [ -f "/sys/module/kvm_intel/parameters/nested" ]; then
+        kvm_nested=$( cat /sys/module/kvm_intel/parameters/nested )
+    elif [ -f "/sys/module/kvm_amd/parameters/nested" ]; then
+        kvm_nested=$( cat /sys/module/kvm_amd/parameters/nested )
+    fi
+    [ "$kvm_nested" == "1" ] || [ "$kvm_nested" == "Y" ] || [ "$kvm_nested" == "y" ]
+}
+
+function install_kubevirt() {
+    for node in $(kubectl get node --no-headers  -o custom-columns=":metadata.name"); do
+        $OCI_BIN exec -t $node bash -c "echo 'fs.inotify.max_user_watches=1048576' >> /etc/sysctl.conf"
+        $OCI_BIN exec -t $node bash -c "echo 'fs.inotify.max_user_instances=512' >> /etc/sysctl.conf"
+        $OCI_BIN exec -i $node bash -c "sysctl -p /etc/sysctl.conf"
+        if [[ "${node}" =~ worker ]]; then
+            kubectl label nodes $node node-role.kubernetes.io/worker="" --overwrite=true
+        fi
+    done
+    local nightly_build_base_url="https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt"
+    local latest=$(curl -sL "${nightly_build_base_url}/latest")
+
+    echo "Deploy latest nighly build Kubevirt"
+    if [ "$(kubectl get kubevirts -n kubevirt kubevirt -ojsonpath='{.status.phase}')" != "Deployed" ]; then
+      kubectl apply -f "${nightly_build_base_url}/${latest}/kubevirt-operator.yaml"
+      kubectl apply -f "${nightly_build_base_url}/${latest}/kubevirt-cr.yaml"
+      if ! is_nested_virt_enabled; then
+        kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
+      fi
+    fi
+    if ! kubectl wait -n kubevirt kv kubevirt --for condition=Available --timeout 15m; then
+        kubectl get pod -n kubevirt -l || true
+        kubectl describe pod -n kubevirt -l || true
+        for p in $(kubectl get pod -n kubevirt -l -o name |sed "s#pod/##"); do
+            kubectl logs -p --all-containers=true -n kubevirt $p || true
+            kubectl logs --all-containers=true -n kubevirt $p || true
+        done
+    fi
+}
+
 check_dependencies
 # In order to allow providing arguments with spaces, e.g. "-vconsole:info -vfile:info"
 # the original command <parse_args $*> was replaced by <parse_args "$@">
@@ -1287,4 +1339,7 @@ if [ "$KIND_INSTALL_METALLB" == true ]; then
 fi
 if [ "$KIND_INSTALL_PLUGINS" == true ]; then
   install_plugins
-fi
+fi
+if [ "$KIND_INSTALL_KUBEVIRT" == true ]; then
+  install_kubevirt
+fi
diff --git a/dist/images/Dockerfile.fedora.dev b/dist/images/Dockerfile.fedora.dev
@@ -28,6 +28,7 @@ RUN INSTALL_PKGS=" \
     python3-openvswitch python3-pyOpenSSL \
     autoconf automake libtool g++ gcc fedora-packager rpmdevtools \
     unbound unbound-devel groff python3-sphinx graphviz openssl openssl-devel \
+    libbpf-devel libxdp-devel numactl-devel \
     checkpolicy libcap-ng-devel selinux-policy-devel" && \
     dnf install --best --refresh -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
     dnf clean all && rm -rf /var/cache/dnf/*

diff --git a/go-controller/go.mod b/go-controller/go.mod
@@ -49,6 +49,7 @@ require (
 	k8s.io/client-go v0.27.1
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/utils v0.0.0-20230505201702-9f6742963106
+	kubevirt.io/api v1.0.0-alpha.0
 )
 
 require (
@@ -85,6 +86,8 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/openshift/custom-resource-status v1.1.2 // indirect
+	github.com/pborman/uuid v1.2.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
@@ -105,7 +108,10 @@ require (
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.23.17 // indirect
 	k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect
+	kubevirt.io/containerized-data-importer-api v1.55.2 // indirect
+	kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect