diff --git a/Dockerfile b/Dockerfile index aa82bcd4..39b3bca9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,31 +6,31 @@ # 2019-11-29T07:08:08Z ARG ROOTLESSKIT_COMMIT=8cf0679be24c640267784f500c65ace2b44b0412 -# 2019-11-21T20:14:45Z -ARG SLIRP4NETNS_COMMIT=21fdece2737dc24ffa3f01a341b8a6854f8b13b4 -# 2019-12-02T15:10:37Z -ARG RUNC_COMMIT=c35c2c9cec6ee503ef31edbaddac9617247ec328 -# 2019-11-27T22:20:17Z -ARG MOBY_COMMIT=3152f9436292115c97b4d8bb18c66cf97876ee75 -# 2019-12-03T02:07:39Z -ARG CONTAINERD_COMMIT=8b12d46a395ae3eed3cd718a7bcc721405f650d7 -# 2019-11-28T12:50:09Z -ARG CRIO_COMMIT=724513d4b7cd923881a05eb90ce62ad3af3f59b6 -# 2019-11-13T16:20:45Z -ARG CNI_PLUGINS_COMMIT=497560f35f2cef2695f1690137b0bba98adf849b -# 2019-12-03T06:56:57Z -ARG KUBERNETES_COMMIT=95a3cd54cf739019b1211163add7247bd31c0ed7 +# 2019-12-18T03:10:18Z +ARG SLIRP4NETNS_COMMIT=a8414d1d1629f6f7a93b60b55e183a93d10d9a1c +# 2019-12-26T15:41:07Z +ARG RUNC_COMMIT=a88592a63474e6976030b4fbded41dd445152236 +# 2019-12-31T04:43:51Z +ARG MOBY_COMMIT=8ca8f8bd6598d44d344da407663a57584a0dc6a2 +# 2019-12-31T05:02:27Z +ARG CONTAINERD_COMMIT=537afb149869e5ef6acbd267f9b02c648abcbef3 +# 2019-12-24T20:35:55Z +ARG CRIO_COMMIT=2dc77fc03c522237a353ca8dfbd9a69c1319bed6 +# 2019-12-18T16:16:58Z +ARG CNI_PLUGINS_COMMIT=ec8f6c99d030bd75337ae8bfc62fc02cdc462528 +# 2019-12-31T07:15:39Z +ARG KUBERNETES_COMMIT=36db62cd7397d153d037bcc4eec34dce99b03fc6 # Version definitions (cont.) -ARG CONMON_RELEASE=v2.0.3 +ARG CONMON_RELEASE=v2.0.8 ARG DOCKER_CLI_RELEASE=19.03.5 # Kube's build script requires KUBE_GIT_VERSION to be set to a semver string ARG KUBE_GIT_VERSION=v1.18.0-usernetes -ARG BAZEL_RELEASE=1.2.1 +ARG BAZEL_RELEASE=2.0.0 ARG SOCAT_RELEASE=tag-1.7.3.3 ARG FLANNEL_RELEASE=v0.11.0 ARG ETCD_RELEASE=v3.4.3 -ARG GOTASK_RELEASE=v2.7.1 +ARG GOTASK_RELEASE=v2.8.0 ARG BASEOS=ubuntu @@ -108,7 +108,7 @@ RUN git pull && git checkout ${CONTAINERD_COMMIT} # workaround: https://github.com/containerd/containerd/issues/3646 RUN ./script/setup/install-dev-tools RUN make EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"' BUILDTAGS="netgo osusergo static_build" && \ - mkdir /out && cp bin/containerd bin/containerd-shim bin/containerd-shim-runc-v1 bin/ctr /out + mkdir /out && cp bin/containerd bin/containerd-shim bin/containerd-shim-runc-v1 bin/containerd-shim-runc-v2 bin/ctr /out ### CRI-O (crio-build) # We don't use Alpine here so as to build cri-o linked with glibc rather than musl libc. @@ -119,7 +119,7 @@ RUN git clone https://github.com/cri-o/cri-o.git /go/src/github.com/cri-o/cri-o WORKDIR /go/src/github.com/cri-o/cri-o ARG CRIO_COMMIT RUN git pull && git checkout ${CRIO_COMMIT} -RUN make binaries && mkdir /out && cp bin/crio /out +RUN make binaries && mkdir /out && cp bin/crio bin/crio-status bin/pinns /out ### conmon (conmon-build) FROM common-golang-alpine-heavy AS conmon-build diff --git a/boot/crio.sh b/boot/crio.sh index 9282f869..b03ff05d 100755 --- a/boot/crio.sh +++ b/boot/crio.sh @@ -4,7 +4,7 @@ source $U7S_BASE_DIR/common/common.inc.sh nsenter::main $0 $@ export _CRIO_ROOTLESS=1 -mkdir -p $XDG_DATA_HOME/containers $XDG_CONFIG_HOME/containers $XDG_CONFIG_HOME/crio $XDG_CONFIG_HOME/crio/runc +mkdir -p $XDG_DATA_HOME/containers/oci/hooks.d $XDG_CONFIG_HOME/containers $XDG_CONFIG_HOME/crio $XDG_CONFIG_HOME/crio/runc # It looks like both crio.conf["registries"] and --registry CLI flags are needed # https://trello.com/c/kmdF350I/521-8-registry-patch-in-cri-o @@ -31,6 +31,7 @@ exec crio \ --cni-config-dir /etc/cni/net.d \ --cni-plugin-dir /opt/cni/bin \ --root $XDG_DATA_HOME/containers/storage \ + --hooks-dir $XDG_DATA_HOME/containers/oci/hooks.d \ --cgroup-manager=cgroupfs \ --storage-driver vfs \ $@ diff --git a/boot/etcd-init-data.sh b/boot/etcd-init-data.sh index e3196337..60291b39 100755 --- a/boot/etcd-init-data.sh +++ b/boot/etcd-init-data.sh @@ -7,5 +7,5 @@ nsenter::main $0 $@ if [[ $U7S_FLANNEL == 1 ]]; then config=$U7S_BASE_DIR/config/flannel/etcd/coreos.com_network_config set -x - timeout 60 sh -c "until cat $config | etcdctl set /coreos.com/network/config; do sleep 1; done" + timeout 60 sh -c "until cat $config | ETCDCTL_API=2 etcdctl set /coreos.com/network/config; do sleep 1; done" fi diff --git a/boot/etcd.sh b/boot/etcd.sh index 129164a7..b81cc08f 100755 --- a/boot/etcd.sh +++ b/boot/etcd.sh @@ -2,4 +2,4 @@ export U7S_BASE_DIR=$(realpath $(dirname $0)/..) source $U7S_BASE_DIR/common/common.inc.sh -exec $(dirname $0)/nsenter.sh etcd --data-dir $XDG_DATA_HOME/usernetes/etcd $@ +exec $(dirname $0)/nsenter.sh etcd --data-dir $XDG_DATA_HOME/usernetes/etcd --enable-v2=true $@ diff --git a/config/flannel/cni_net.d/10-flannel.conflist b/config/flannel/cni_net.d/10-flannel.conflist index 9c176653..cc2f213c 100644 --- a/config/flannel/cni_net.d/10-flannel.conflist +++ b/config/flannel/cni_net.d/10-flannel.conflist @@ -1,4 +1,5 @@ { + "cniVersion": "0.4.0", "name": "cbr0", "plugins": [ { diff --git a/hack/smoketest.sh b/hack/smoketest.sh index 799c1152..ef04b872 100755 --- a/hack/smoketest.sh +++ b/hack/smoketest.sh @@ -17,9 +17,11 @@ function cleanup() { } trap cleanup EXIT docker exec $container ./boot/nsenter.sh echo rootlesskit ready -timeout 60 sh -ex -c "until docker exec $container ./kubectl.sh get nodes; do sleep 5; done" +timeout 60 sh -ex -c "until test \$(docker exec $container ./kubectl.sh get nodes -o name | grep ^node/ | wc -l) -gt 0; do sleep 5; done" || docker logs $container function k(){ docker exec -it $container ./kubectl.sh $@ } +k get nodes -o wide k get nodes -o yaml k run --rm -i --image busybox --restart=Never hello echo hello $container +k get nodes -o wide diff --git a/src/patches/kubernetes/0001-kubelet-cm-ignore-sysctl-error-when-running-in-usern.patch b/src/patches/kubernetes/0001-kubelet-cm-ignore-sysctl-error-when-running-in-usern.patch index eba2cdd2..1600634f 100644 --- a/src/patches/kubernetes/0001-kubelet-cm-ignore-sysctl-error-when-running-in-usern.patch +++ b/src/patches/kubernetes/0001-kubelet-cm-ignore-sysctl-error-when-running-in-usern.patch @@ -1,4 +1,4 @@ -From d80b6f413e5059edc4cce2548ec19a556d964608 Mon Sep 17 00:00:00 2001 +From e87123e8a887a39b478af7869a050c6a929be6f4 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Tue, 21 Aug 2018 16:45:04 +0900 Subject: [PATCH 1/3] kubelet/cm: ignore sysctl error when running in userns @@ -10,10 +10,10 @@ Signed-off-by: Akihiro Suda 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD -index 7b57d09c324..f70ede1bb30 100644 +index edcc7f53bf6..723197f1dd3 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD -@@ -71,6 +71,7 @@ go_library( +@@ -72,6 +72,7 @@ go_library( "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/systemd:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/configs:go_default_library", @@ -21,7 +21,7 @@ index 7b57d09c324..f70ede1bb30 100644 "//vendor/k8s.io/utils/io:go_default_library", "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/path:go_default_library", -@@ -121,6 +122,7 @@ go_library( +@@ -123,6 +124,7 @@ go_library( "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/systemd:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/configs:go_default_library", @@ -30,7 +30,7 @@ index 7b57d09c324..f70ede1bb30 100644 "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/path:go_default_library", diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go -index 81d3a015639..3653bf7124e 100644 +index 90ad0c1bb49..bf3e2a725d4 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -32,6 +32,7 @@ import ( @@ -41,7 +41,7 @@ index 81d3a015639..3653bf7124e 100644 "k8s.io/klog" utilio "k8s.io/utils/io" "k8s.io/utils/mount" -@@ -414,7 +415,11 @@ func setupKernelTunables(option KernelTunableBehavior) error { +@@ -415,7 +416,11 @@ func setupKernelTunables(option KernelTunableBehavior) error { klog.V(2).Infof("Updating kernel flag: %v, expected value: %v, actual value: %v", flag, expectedValue, val) err = sysctl.SetSysctl(flag, expectedValue) if err != nil { diff --git a/src/patches/kubernetes/0002-kube-proxy-allow-running-in-userns.patch b/src/patches/kubernetes/0002-kube-proxy-allow-running-in-userns.patch index 10e9caa1..e6ac7259 100644 --- a/src/patches/kubernetes/0002-kube-proxy-allow-running-in-userns.patch +++ b/src/patches/kubernetes/0002-kube-proxy-allow-running-in-userns.patch @@ -1,4 +1,4 @@ -From f6fcbe86caf7ece92cd46577b6c33db161482962 Mon Sep 17 00:00:00 2001 +From 9430ea57cef63808fbe9b0f4e2f50e761b4e4b25 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 23 Aug 2018 14:14:44 +0900 Subject: [PATCH 2/3] kube-proxy: allow running in userns @@ -12,99 +12,99 @@ Signed-off-by: Akihiro Suda 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD -index bbabff37d6e..9c2226dcb3d 100644 +index 6cfc9fe681d..4500164ab73 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD -@@ -82,6 +82,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -83,6 +83,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ -@@ -89,6 +90,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -91,6 +92,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:dragonfly": [ -@@ -96,6 +98,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -99,6 +101,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ -@@ -103,6 +106,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -107,6 +110,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:ios": [ -@@ -110,6 +114,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -115,6 +119,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ -@@ -117,6 +122,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -123,6 +128,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ -@@ -124,6 +130,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -131,6 +137,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:netbsd": [ -@@ -131,6 +138,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -139,6 +146,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:openbsd": [ -@@ -138,6 +146,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -147,6 +155,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:plan9": [ -@@ -145,6 +154,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -155,6 +164,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:solaris": [ -@@ -152,6 +162,7 @@ go_library( - "//pkg/util/node:go_default_library", +@@ -163,6 +173,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:windows": [ diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go -index 9c5cb19f236..4bd3687860d 100644 +index 6095e81e599..96b27232769 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -26,6 +26,7 @@ import ( @@ -115,7 +115,7 @@ index 9c5cb19f236..4bd3687860d 100644 v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" -@@ -249,6 +250,12 @@ func newProxyServer( +@@ -260,6 +261,12 @@ func newProxyServer( } } @@ -128,7 +128,7 @@ index 9c5cb19f236..4bd3687860d 100644 return &ProxyServer{ Client: client, EventClient: eventClient, -@@ -260,7 +267,7 @@ func newProxyServer( +@@ -271,7 +278,7 @@ func newProxyServer( Broadcaster: eventBroadcaster, Recorder: recorder, ConntrackConfiguration: config.Conntrack, diff --git a/src/patches/kubernetes/0003-kubelet-new-feature-gate-SupportNoneCgroupDriver.patch b/src/patches/kubernetes/0003-kubelet-new-feature-gate-SupportNoneCgroupDriver.patch index 7a05b2ed..5f280d12 100644 --- a/src/patches/kubernetes/0003-kubelet-new-feature-gate-SupportNoneCgroupDriver.patch +++ b/src/patches/kubernetes/0003-kubelet-new-feature-gate-SupportNoneCgroupDriver.patch @@ -1,4 +1,4 @@ -From 0a932526ab754add7ff14fd419f508d43196e271 Mon Sep 17 00:00:00 2001 +From c77b971794a6998f3a59e7a67ad1f43c2ffcee30 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sun, 2 Jun 2019 18:39:05 +0900 Subject: [PATCH 3/3] kubelet: new feature gate: SupportNoneCgroupDriver @@ -103,10 +103,10 @@ index 8a705bb139a..2aa530257a4 100644 if kubeDeps.CAdvisorInterface == nil { diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go -index 1cf35280df3..b8a45fe21d9 100644 +index 309dbb29556..be6f9013c9f 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go -@@ -547,6 +547,18 @@ const ( +@@ -548,6 +548,18 @@ const ( // // Enables topology aware service routing ServiceTopology featuregate.Feature = "ServiceTopology" @@ -125,7 +125,7 @@ index 1cf35280df3..b8a45fe21d9 100644 ) func init() { -@@ -633,6 +645,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS +@@ -634,6 +646,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS AllowInsecureBackendProxy: {Default: true, PreRelease: featuregate.Beta}, PodDisruptionBudget: {Default: true, PreRelease: featuregate.Beta}, ServiceTopology: {Default: false, PreRelease: featuregate.Alpha}, @@ -260,10 +260,10 @@ index 5d77ed7a45f..5654d737fd6 100644 func (m *unsupportedCgroupManager) Name(_ CgroupName) string { diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go -index 3653bf7124e..50b76c5426a 100644 +index bf3e2a725d4..7b115ec91f1 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go -@@ -248,9 +248,15 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I +@@ -249,9 +249,15 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I // Turn CgroupRoot from a string (in cgroupfs path format) to internal CgroupName cgroupRoot := ParseCgroupfsToCgroupName(nodeConfig.CgroupRoot) @@ -280,7 +280,7 @@ index 3653bf7124e..50b76c5426a 100644 // this does default to / when enabled, but this tests against regressions. if nodeConfig.CgroupRoot == "" { return nil, fmt.Errorf("invalid configuration: cgroups-per-qos was specified and cgroup-root was not specified. To enable the QoS cgroup hierarchy you need to specify a valid cgroup-root") -@@ -260,7 +266,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I +@@ -261,7 +267,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I // of note, we always use the cgroupfs driver when performing this check since // the input is provided in that format. // this is important because we do not want any name conversion to occur.