Windows Server run with --cni got [ctr: no network config found in /etc/cni/net.d: cni plugin not initialized] error

[kkbruce]

I create a new Windows Server Core 2022 VM and follow Get started: Prep Windows for containers - Windows Server - Containerd script

Invoke-WebRequest -UseBasicParsing "https://raw.githubusercontent.com/microsoft/Windows-Containers/Main/helpful_tools/Install-ContainerdRuntime/install-containerd-runtime.ps1" -o install-containerd-runtime.ps1
.\install-containerd-runtime.ps1

Run script log:

PS C:\> .\install-containerd-runtime.ps1
Querying status of Windows feature: Containers...
Feature Containers is already enabled.
Downloading containerd, nerdCTL, and Windows CNI binaries...
x bin/
x bin/containerd.exe
x bin/containerd-shim-runhcs-v1.exe
x bin/containerd-stress.exe
x bin/ctr.exe
Containerd binaries added to C:\Program Files\containerd
x nerdctl.exe
NerdCTL binary added to C:\Program Files\nerdctl
x sdnoverlay.exe
x sdnbridge.exe
x nat.exe
CNI plugin binaries added to C:\Program Files\containerd\cni\bin
Adding C:\Program Files\containerd, C:\Program Files\nerdctl, C:\Program Files\containerd\cni\bin to the path
Configuring the containerd service
disabled_plugins = []
imports = []
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "C:\\ProgramData\\containerd\\root"
state = "C:\\ProgramData\\containerd\\state"
temp = ""
version = 2

[cgroup]
  path = ""

[debug]
  address = ""
  format = ""
  gid = 0
  level = ""
  uid = 0

[grpc]
  address = "\\\\.\\pipe\\containerd-containerd"
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216
  tcp_address = ""
  tcp_tls_ca = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0

[metrics]
  address = ""
  grpc_histogram = false

[plugins]

  [plugins."io.containerd.gc.v1.scheduler"]
    deletion_threshold = 0
    mutation_threshold = 100
    pause_threshold = 0.02
    schedule_delay = "0s"
    startup_delay = "100ms"

  [plugins."io.containerd.grpc.v1.cri"]
    device_ownership_from_security_context = false
    disable_apparmor = false
    disable_cgroup = false
    disable_hugetlb_controller = false
    disable_proc_mount = false
    disable_tcp_service = true
    enable_selinux = false
    enable_tls_streaming = false
    enable_unprivileged_icmp = false
    enable_unprivileged_ports = false
    ignore_image_defined_volumes = false
    max_concurrent_downloads = 3
    max_container_log_line_size = 16384
    netns_mounts_under_state_dir = false
    restrict_oom_score_adj = false
    sandbox_image = "k8s.gcr.io/pause:3.6"
    selinux_category_range = 0
    stats_collect_period = 10
    stream_idle_timeout = "4h0m0s"
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    systemd_cgroup = false
    tolerate_missing_hugetlb_controller = false
    unset_seccomp_profile = ""

    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "C:\\Program Files\\containerd\\cni\\bin"
      conf_dir = "C:\\Program Files\\containerd\\cni\\conf"
      conf_template = ""
      ip_pref = ""
      max_conf_num = 1

    [plugins."io.containerd.grpc.v1.cri".containerd]
      default_runtime_name = "runhcs-wcow-process"
      disable_snapshot_annotations = false
      discard_unpacked_layers = false
      ignore_rdt_not_enabled_errors = false
      no_pivot = false
      snapshotter = "windows"

      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        base_runtime_spec = ""
        cni_conf_dir = ""
        cni_max_conf_num = 0
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_path = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]

      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]

        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process]
          base_runtime_spec = ""
          cni_conf_dir = ""
          cni_max_conf_num = 0
          container_annotations = []
          pod_annotations = []
          privileged_without_host_devices = false
          runtime_engine = ""
          runtime_path = ""
          runtime_root = ""
          runtime_type = "io.containerd.runhcs.v1"

          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process.options]

      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        base_runtime_spec = ""
        cni_conf_dir = ""
        cni_max_conf_num = 0
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_path = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]

    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = "node"

    [plugins."io.containerd.grpc.v1.cri".registry]
      config_path = ""

      [plugins."io.containerd.grpc.v1.cri".registry.auths]

      [plugins."io.containerd.grpc.v1.cri".registry.configs]

      [plugins."io.containerd.grpc.v1.cri".registry.headers]

      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]

    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""

  [plugins."io.containerd.internal.v1.opt"]
    path = "C:\\ProgramData\\containerd\\root\\opt"

  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"

  [plugins."io.containerd.internal.v1.tracing"]
    sampling_ratio = 1.0
    service_name = "containerd"

  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"

  [plugins."io.containerd.runtime.v2.task"]
    platforms = ["windows/amd64", "linux/amd64"]
    sched_core = false

  [plugins."io.containerd.service.v1.diff-service"]
    default = ["windows", "windows-lcow"]

  [plugins."io.containerd.service.v1.tasks-service"]
    rdt_config_file = ""

  [plugins."io.containerd.tracing.processor.v1.otlp"]
    endpoint = ""
    insecure = false
    protocol = ""

[proxy_plugins]

[stream_processors]

  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
    args = ["--decryption-keys-path", "C:\\Program Files\\containerd\\ocicrypt\\keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=C:\\Program Files\\containerd\\ocicrypt\\ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar"

  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
    args = ["--decryption-keys-path", "C:\\Program Files\\containerd\\ocicrypt\\keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=C:\\Program Files\\containerd\\ocicrypt\\ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar+gzip"

[timeouts]
  "io.containerd.timeout.bolt.open" = "0s"
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[ttrpc]
  address = ""
  gid = 0
  uid = 0
Waiting for Containerd daemon...
Successfully connected to Containerd Daemon.
The following images are present on this machine:
REPOSITORY    TAG    IMAGE ID    CREATED    PLATFORM    SIZE    BLOB SIZE
Script complete!

Try run without --cni is work and with --cni is failing, error message: 「ctr: no network config found in /etc/cni/net.d: cni plugin not initialized」, see picture:

Search solution and create a nat network. (The install-containerd-runtime.ps1 does not set this part.)

curl.exe -LO https://raw.githubusercontent.com/microsoft/SDN/master/Kubernetes/windows/hns.psm1
ipmo ./hns.psm1

$subnet="10.0.0.0/16" 
$gateway="10.0.0.1"
New-HNSNetwork -Type Nat -AddressPrefix $subnet -Gateway $gateway -Name "nat"

Set up the Containerd network config using the same gateway and subnet.

@"
{
    "cniVersion": "0.2.0",
    "name": "nat",
    "type": "nat",
    "master": "Ethernet",
    "ipam": {
        "subnet": "$subnet",
        "routes": [
            {
                "gateway": "$gateway"
            }
        ]
    },
    "capabilities": {
        "portMappings": true,
        "dns": true
    }
}
"@ | Set-Content "C:\Program Files\containerd\cni\conf\0-containerd-nat.conf" -Force

It will get the same error message with --cni .

[kkbruce]

Refer: containerd/nerdctl#559 (comment)
Refer: containerd/containerd#6304 (comment)

Create c:\etc\cni directory

PS C:\etc> tree
Folder PATH listing for volume Windows
Volume serial number is 94B1-2EBA
C:.
└───cni
    └───net.d

and create an SYMLINKD net.d [C:\Program Files\containerd\cni\conf]

C:\etc\cni>dir
 Volume in drive C is Windows
 Volume Serial Number is 94B1-2EBA

 Directory of C:\etc\cni

07/15/2022  02:05 AM    <DIR>          .
07/15/2022  02:05 AM    <DIR>          ..
07/15/2022  02:05 AM    <SYMLINKD>     net.d [C:\Program Files\containerd\cni\conf]

Same step create c:\opt\cni and symlinks bin [C:\Program Files\containerd\cni\bin], like:

C:\opt\cni>dir
 Volume in drive C is Windows
 Volume Serial Number is 94B1-2EBA

 Directory of C:\opt\cni

07/15/2022  02:05 AM    <DIR>          .
07/15/2022  02:05 AM    <DIR>          ..
07/15/2022  02:05 AM    <SYMLINKD>     bin [C:\Program Files\containerd\cni\bin]

run restart-service containerd and test --cni will work.

PS C:\> ctr run --cni --rm mcr.microsoft.com/windows/nanoserver:ltsc2022 test ipconfig

Windows IP Configuration


Ethernet adapter vEthernet (default-test_nat):

   Connection-specific DNS Suffix  . :
   Link-local IPv6 Address . . . . . : fe80::1c4a:2067:bd2e:b085%17
   IPv4 Address. . . . . . . . . . . : 172.20.86.46
   Subnet Mask . . . . . . . . . . . : 255.255.240.0
   Default Gateway . . . . . . . . . : 172.20.80.1

[lippertmarkus]

The symlink setup is only needed for ctr to work, but isn't required for others like crictl or kubelet and that's why that isn't part of the runtime setup script. ctr paths have been changed in https://github.com/containerd/go-cni/pull/103/files#diff-df2b994645f533d6ced46ef610f95cfc41a3cecc099a873a9b47fadccb605b9b