feat: [dropgz] Dropgz for windows

.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml

@@ -15,6 +15,8 @@ stages:
     displayName: "Windows AKS Cluster ${{ parameters.cni }}"
     jobs:
       - job: create_aks_cluster_with_${{ parameters.name }}
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
           - template: ../load-test-templates/create-cluster-template.yaml
             parameters:
@@ -32,10 +34,10 @@ stages:
           name: "$(BUILD_POOL_NAME_LINUX_AMD64)"
         strategy:
           matrix:
-            windows_cniv1_amd64:
-              os: windows
-              name: cni-plugin
+            cni_dropgz_windows2022_amd64:
               arch: amd64
+              name: cni-dropgz-test
+              os: windows
               os_version: ltsc2022
         steps:
           - template: ../../containers/container-template.yaml
@@ -45,15 +47,17 @@ stages:
               os: $(os)
               os_version: $(os_version)
   - stage:  update_cni
-    dependsOn: 
+    dependsOn:
       - createAKSclusterWindows
       - build_images
     displayName: "Update CNI on Cluster"
     jobs:
       - job: deploy_pods
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
         strategy:
           matrix:
-            windows_cniv1_amd64:
+            cni_dropgz_windows2022_amd64:
               os: windows
               arch: amd64
               os_version: ltsc2022
@@ -66,12 +70,12 @@ stages:
               addSpnToEnvironment: true
               inlineScript: |
                 set -ex
-                export CNI_IMAGE=$(make cni-plugin-image-name-and-tag OS=$(os) ARCH=$(arch) OS_VERSION=$(os_version))
-                az extension add --name aks-preview
                 clusterName=${{ parameters.clusterName }}-$(make revision)
                 make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
                 make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
-                envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f -
+                dropgz_version=$(make cni-dropgz-version)
+                export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS=$(os) ARCH=$(arch) OS_VERSION=$(os_version) CNI_DROPGZ_VERSION=${dropgz_version})
+                envsubst < ./test/integration/manifests/cni/cni-installer-v1-windows.yaml | kubectl apply -f -
             name: "UploadCNI"
             displayName: "Upload CNI"
           - script: |
@@ -85,6 +89,8 @@ stages:
     displayName: "Pod Deployment"
     jobs:
       - job: deploy_pods
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
           - template: ../load-test-templates/pod-deployment-template.yaml
             parameters:
@@ -98,6 +104,8 @@ stages:
     displayName: "Validate State"
     jobs:
       - job: validate_state
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
@@ -110,6 +118,8 @@ stages:
       - validate_state_windows
     jobs:
       - job: delete_resources
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
           - task: AzureCLI@1
             inputs:

.pipelines/pipeline.yaml

@@ -167,16 +167,11 @@ stages:
               arch: amd64
               name: cni-dropgz-test
               os: linux
-            cni_plugin_windows2022_amd64:
-              os: windows
-              name: cni-plugin
+            cni_dropgz_test_windows2022_amd64:
               arch: amd64
-              os_version: ltsc2022
-            cni_plugin_windows2019_amd64:
+              name: cni-dropgz-test
               os: windows
-              name: cni-plugin
-              arch: amd64
-              os_version: ltsc2019
+              os_version: ltsc2022
             cns_linux_amd64:
               arch: amd64
               name: cns
@@ -289,7 +284,7 @@ stages:
               platforms: linux/amd64 linux/arm64
             cni_dropgz_test:
               name: cni-dropgz-test
-              platforms: linux/amd64 linux/arm64
+              platforms: linux/amd64 linux/arm64 windows/amd64
             cns:
               name: cns
               os_versions: ltsc2019 ltsc2022

.pipelines/singletenancy/aks/e2e-step-template.yaml

@@ -44,8 +44,9 @@ steps:
         export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='linux' ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }})
         envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f -
         kubectl rollout status daemonset/azure-cni -n kube-system
-        export CNI_IMAGE=$(make cni-plugin-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} OS_VERSION=${{ parameters.os_version }} CNI_VERSION=${{ parameters.cniVersion }})
-        envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f -
+        echo "Deploying on windows nodes"
+        export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='windows' ARCH=${{ parameters.arch }}  OS_VERSION=${{ parameters.os_version }} CNI_DROPGZ_VERSION=${{ parameters.version }})
+        envsubst < ./test/integration/manifests/cni/cni-installer-v1-windows.yaml | kubectl apply -f -
         kubectl rollout status daemonset/azure-cni-windows -n kube-system
       else
         export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }})

Makefile

@@ -343,7 +343,7 @@ cni-dropgz-image-name-and-tag: # util target to print the CNI dropgz image name
 cni-dropgz-image: ## build cni-dropgz container image.
 	$(MAKE) container \
 		DOCKERFILE=dropgz/build/$(OS).Dockerfile \
-		EXTRA_BUILD_ARGS='--build-arg OS=$(OS) --build-arg ARCH=$(ARCH)' \
+		EXTRA_BUILD_ARGS='--build-arg OS=$(OS) --build-arg ARCH=$(ARCH) --build-arg OS_VERSION=$(OS_VERSION)' \
 		IMAGE=$(CNI_DROPGZ_IMAGE) \
 		TAG=$(CNI_DROPGZ_PLATFORM_TAG)
 
@@ -367,8 +367,8 @@ cni-dropgz-test-image-name-and-tag: # util target to print the CNI dropgz test i
 
 cni-dropgz-test-image: ## build cni-dropgz-test container image.
 	$(MAKE) container \
-		DOCKERFILE=dropgz/build/cniTest.Dockerfile \
-		EXTRA_BUILD_ARGS='--build-arg OS=$(OS)' \
+		DOCKERFILE=dropgz/build/cniTest_$(OS).Dockerfile \
+		EXTRA_BUILD_ARGS='--build-arg OS=$(OS)  --build-arg ARCH=$(ARCH) --build-arg OS_VERSION=$(OS_VERSION)' \
 		IMAGE=$(CNI_DROPGZ_TEST_IMAGE) \
 		TAG=$(CNI_DROPGZ_TEST_PLATFORM_TAG)
 

dropgz/build/cniTest_windows.Dockerfile

@@ -0,0 +1,29 @@
+ARG ARCH
+ARG OS_VERSION
+FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang:1.20 AS azure-vnet
+ARG VERSION
+WORKDIR /azure-container-networking
+COPY . .
+RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/plugin/main.go
+RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet-telemetry.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/telemetry/service/telemetrymain.go
+RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet-ipam.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/ipam/plugin/main.go
+
+FROM --platform=linux/${ARCH} mcr.microsoft.com/cbl-mariner/base/core:2.0 AS compressor
+ARG OS
+WORKDIR /dropgz
+COPY dropgz .
+COPY --from=azure-vnet /azure-container-networking/azure-vnet.exe pkg/embed/fs
+COPY --from=azure-vnet /azure-container-networking/azure-vnet-telemetry.exe pkg/embed/fs
+COPY --from=azure-vnet /azure-container-networking/azure-vnet-ipam.exe pkg/embed/fs
+RUN cd pkg/embed/fs/ && sha256sum * > sum.txt
+RUN gzip --verbose --best --recursive pkg/embed/fs && for f in pkg/embed/fs/*.gz; do mv -- "$f" "${f%%.gz}"; done
+
+FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang:1.20 AS dropgz
+ARG VERSION
+WORKDIR /dropgz
+COPY --from=compressor /dropgz .
+RUN GOOS=windows CGO_ENABLED=0 go build -a -o bin/dropgz.exe -trimpath -ldflags "-X github.com/Azure/azure-container-networking/dropgz/internal/buildinfo.Version="$VERSION"" -gcflags="-dwarflocationlists=true" main.go
+
+FROM mcr.microsoft.com/windows/nanoserver:${OS_VERSION}
+COPY --from=dropgz /dropgz/bin/dropgz.exe dropgz.exe
+ENTRYPOINT [ "dropgz.exe" ]

dropgz/build/windows.Dockerfile

@@ -0,0 +1,35 @@
+ARG ARCH
+ARG OS_VERSION
+FROM --platform=linux/${ARCH} mcr.microsoft.com/cbl-mariner/base/core:2.0 AS tar
+RUN tdnf install -y tar
+RUN tdnf install -y unzip
+RUN tdnf upgrade -y && tdnf install -y ca-certificates
+
+FROM tar AS azure-vnet
+ARG AZCNI_VERSION=v1.5.4
+ARG VERSION
+ARG OS
+ARG ARCH
+WORKDIR /azure-container-networking
+COPY . .
+RUN curl -LO --cacert /etc/ssl/certs/ca-certificates.crt https://github.com/Azure/azure-container-networking/releases/download/$AZCNI_VERSION/azure-vnet-cni-$OS-$ARCH-$AZCNI_VERSION.zip && unzip -o azure-vnet-cni-$OS-$ARCH-$AZCNI_VERSION.zip
+
+FROM --platform=linux/${ARCH} mcr.microsoft.com/cbl-mariner/base/core:2.0 AS compressor
+ARG OS
+WORKDIR /dropgz
+COPY dropgz .
+COPY --from=azure-vnet /azure-container-networking/azure-vnet.exe pkg/embed/fs
+COPY --from=azure-vnet /azure-container-networking/azure-vnet-telemetry.exe pkg/embed/fs
+COPY --from=azure-vnet /azure-container-networking/azure-vnet-ipam.exe pkg/embed/fs
+RUN cd pkg/embed/fs/ && sha256sum * > sum.txt
+RUN gzip --verbose --best --recursive pkg/embed/fs && for f in pkg/embed/fs/*.gz; do mv -- "$f" "${f%%.gz}"; done
+
+FROM --platform=linux/${ARCH} mcr.microsoft.com/oss/go/microsoft/golang:1.20 AS dropgz
+ARG VERSION
+WORKDIR /dropgz
+COPY --from=compressor /dropgz .
+RUN GOOS=windows CGO_ENABLED=0 go build -a -o bin/dropgz.exe -trimpath -ldflags "-X github.com/Azure/azure-container-networking/dropgz/internal/buildinfo.Version="$VERSION"" -gcflags="-dwarflocationlists=true" main.go
+
+FROM mcr.microsoft.com/windows/nanoserver:${OS_VERSION}
+COPY --from=dropgz /dropgz/bin/dropgz.exe dropgz.exe
+ENTRYPOINT [ "dropgz.exe" ]

dropgz/pkg/embed/payload.go

@@ -7,22 +7,24 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"os/exec"
+	"path"
 	"path/filepath"
-	"strings"
+	"runtime"
 
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 )
 
 const (
 	cwd           = "fs"
-	pathPrefix    = cwd + string(filepath.Separator)
 	oldFileSuffix = ".old"
 )
 
 var ErrArgsMismatched = errors.New("mismatched argument count")
 
 // embedfs contains the embedded files for deployment, as a read-only FileSystem containing only "embedfs/".
+//
 //nolint:typecheck // dir is populated at build.
 //go:embed fs
 var embedfs embed.FS
@@ -36,7 +38,8 @@ func Contents() ([]string, error) {
 		if d.IsDir() {
 			return nil
 		}
-		contents = append(contents, strings.TrimPrefix(path, pathPrefix))
+		_, filename := filepath.Split(path)
+		contents = append(contents, filename)
 		return nil
 	})
 	if err != nil {
@@ -69,10 +72,10 @@ func (c *compoundReadCloser) Close() error {
 	return nil
 }
 
-func Extract(path string) (*compoundReadCloser, error) {
-	f, err := embedfs.Open(filepath.Join(cwd, path))
+func Extract(p string) (*compoundReadCloser, error) {
+	f, err := embedfs.Open(path.Join(cwd, p))
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to open file %s", path)
+		return nil, errors.Wrapf(err, "failed to open file %s", p)
 	}
 	r, err := gzip.NewReader(bufio.NewReader(f))
 	if err != nil {
@@ -81,14 +84,29 @@ func Extract(path string) (*compoundReadCloser, error) {
 	return &compoundReadCloser{closer: f, readcloser: r}, nil
 }
 
-func deploy(src, dest string) error {
+func deploy(log *zap.Logger, src, dest string) error {
 	rc, err := Extract(src)
 	if err != nil {
 		return err
 	}
 	defer rc.Close()
 	// check if the file exists at dest already and rename it as an old one
 	if _, err := os.Stat(dest); err == nil {
+		// For windows we need to close the process running with the binary before we can rename it.
+		// This is because the file is locked by the process.
+		// We can't use the os.Rename() function because it will fail with an error.
+		if runtime.GOOS == "windows" {
+			// Get the process image name based on the name of the file
+			processImageName := filepath.Base(dest)
+			cmd := exec.Command("taskkill", "/F", "/IM", processImageName)
+			cmd.Stdout = os.Stdout
+			cmd.Stderr = os.Stderr
+			err := cmd.Run()
+			//Silenty ignore the error if the process is not running
+			if err != nil {
+				log.Info("failed to kill process", zap.String("process", processImageName))
+			}
+		}
 		oldDest := dest + oldFileSuffix
 		if err = os.Rename(dest, oldDest); err != nil {
 			return errors.Wrapf(err, "failed to rename the %s to %s", dest, oldDest)
@@ -110,7 +128,7 @@ func Deploy(log *zap.Logger, srcs, dests []string) error {
 	for i := range srcs {
 		src := srcs[i]
 		dest := dests[i]
-		if err := deploy(src, dest); err != nil {
+		if err := deploy(log, src, dest); err != nil {
 			return err
 		}
 		log.Info("wrote file", zap.String("src", src), zap.String("dest", dest))

hack/manifests/windows-update.yaml → test/integration/manifests/cni/cni-installer-v1-windows.yaml

@@ -36,19 +36,35 @@ spec:
       hostNetwork: true
       serviceAccountName: azure-cni
       tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - effect: NoExecute
-        operator: Exists
-      containers:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - operator: "Exists"
+          effect: NoExecute
+        - operator: "Exists"
+          effect: NoSchedule
+      initContainers:
         - name: cni-drop
-          image: ${CNI_IMAGE}
+          image: ${DROP_GZ_URL}
           imagePullPolicy: Always
+          command: ["%CONTAINER_SANDBOX_MOUNT_POINT%/dropgz.exe"]
+          args:
+            - deploy
+            - azure-vnet.exe
+            - -o
+            - /k/azurecni/bin/azure-vnet.exe
+            - azure-vnet-ipam.exe
+            - -o
+            - /k/azurecni/bin/azure-vnet-ipam.exe
+            - azure-vnet-telemetry.exe
+            - -o
+            - /k/azurecni/bin/azure-vnet-telemetry.exe
           volumeMounts:
             - name: cni-bin
               mountPath: /k/azurecni/bin/
+      containers:
+        - name: pause
+          image: mcr.microsoft.com/oss/kubernetes/pause:3.6
+          command: ["%CONTAINER_SANDBOX_MOUNT_POINT%/pause.exe"]
       volumes:
         - name: cni-bin
           hostPath: