minikube-iso: add cri-o runtime

https://github.com/kubernetes-incubator/cri-o

Updated the runc version to its latest master commit.
Got crio into the automounter to get off the tmpfs

This feature bubbles up to the minikube command by reusing the
`--container-runtime=` flag, by enabling the value of "`crio`"
(`minikube start --container-runtime=crio`), while the flags/config
passed to localkube are more like k8s (`--container-runtime=remote
--remote-runtime-endpoint=/var/run/crio.sock`)

This is mostly ready for review. It is still lacking having
--insecure-registry plumbed through, but for now the policy.json is
open.

Signed-off-by: Vincent Batts <[email protected]>

cmd/localkube/cmd/options.go

@@ -68,6 +68,8 @@ func AddFlags(s *localkube.LocalkubeServer) {
 	flag.Var(&s.RuntimeConfig, "runtime-config", "A set of key=value pairs that describe runtime configuration that may be passed to apiserver. apis/<groupVersion> key can be used to turn on/off specific api versions. apis/<groupVersion>/<resource> can be used to turn on/off specific resources. api/all and api/legacy are special keys to control all and legacy api versions respectively.")
 	flag.IPVar(&s.NodeIP, "node-ip", s.NodeIP, "IP address of the node. If set, kubelet will use this IP address for the node.")
 	flag.StringVar(&s.ContainerRuntime, "container-runtime", "", "The container runtime to be used")
+	flag.StringVar(&s.RemoteRuntimeEndpoint, "remote-runtime-endpoint", "", "The container runtime endpoint (CRI) to be used (if this is set, then --container-runtime is forced as 'remote')")
+	flag.StringVar(&s.RemoteImageEndpoint, "remote-image-endpoint", "", "The container image endpoint (CRI) to be used (if this is set, then --container-runtime is forced as 'remote')")
 	flag.StringVar(&s.NetworkPlugin, "network-plugin", "", "The name of the network plugin")
 	flag.StringVar(&s.FeatureGates, "feature-gates", "", "A set of key=value pairs that describe feature gates for alpha/experimental features.")
 	flag.Var(&s.ExtraConfig, "extra-config", "A set of key=value pairs that describe configuration that may be passed to different components. The key should be '.' separated, and the first part before the dot is the component to apply the configuration to.")

cmd/localkube/cmd/start.go

@@ -61,6 +61,18 @@ func StartLocalkube() {
 }
 
 func SetupServer(s *localkube.LocalkubeServer) {
+	if s.ContainerRuntime == "remote" && s.RemoteRuntimeEndpoint == "" {
+		panic("Failed to connect to --container-runtime='remote' with no --container-runtime-endpoint")
+	}
+	// localkube flags can handle `--container-runtime=remote --remote-runtime-endpoint=/var/run/crio.sock --remote-image-endpoint=/var/run/crio.sock`,
+	// but this allows for a convenience of just e.g.`--container-runtime=crio` and the same for minikube
+	switch s.ContainerRuntime {
+	case "crio", "cri-o":
+		s.ContainerRuntime = "remote"
+		s.RemoteRuntimeEndpoint = "unix:///var/run/crio.sock"
+		s.RemoteImageEndpoint = "unix:///var/run/crio.sock"
+	}
+
 	if s.ShouldGenerateCerts {
 		if err := s.GenerateCerts(); err != nil {
 			fmt.Println("Failed to create certificates!")

deploy/iso/minikube-iso/package/Config.in

@@ -1,5 +1,7 @@
 menu "System tools"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/rkt-bin/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/runc-master/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crio-bin/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/automount/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/docker-bin/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni-bin/Config.in"

deploy/iso/minikube-iso/package/automount/minikube-automount

@@ -185,6 +185,10 @@ if [ -n "$BOOT2DOCKER_DATA" ]; then
     mkdir -p /mnt/$PARTNAME/var/lib/minishift
     mkdir /var/lib/minishift
     mount --bind /mnt/$PARTNAME/var/lib/minishift /var/lib/minishift
+
+    ## make an env file for other services to discover this PARTNAME easier
+    mkdir -p /var/run/minikube
+    echo "PERSISTENT_DIR=\"/mnt/$PARTNAME\"" > /var/run/minikube/env
 fi
 swapon "${UNPARTITIONED_HD}2"
 

deploy/iso/minikube-iso/package/crio-bin/Config.in

@@ -0,0 +1,17 @@
+config BR2_PACKAGE_CRIO_BIN
+	bool "crio-bin"
+	default y
+	depends on BR2_x86_64
+	depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS
+	depends on BR2_PACKAGE_HOST_GO_CGO_LINKING_SUPPORTS
+	depends on BR2_TOOLCHAIN_HAS_THREADS
+	depends on BR2_USE_MMU # lvm2
+	depends on !BR2_STATIC_LIBS # lvm2
+	depends on !BR2_TOOLCHAIN_USES_MUSL # lvm2
+	select BR2_PACKAGE_RUNC_MASTER
+	select BR2_PACKAGE_BTRFS_PROGS
+	select BR2_PACKAGE_LIBSECCOMP
+	select BR2_PACKAGE_LIBGPGME
+	select BR2_PACKAGE_BTRFS_PROGS
+	select BR2_PACKAGE_LVM2
+	select BR2_PACKAGE_LVM2_APP_LIBRARY

deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash

@@ -0,0 +1,2 @@
+#sha256 a4f423aeede371d21a1b32c0f2287ae5d0c4cfb5ed2927e4b3930675b3c8e986 5e3c53c1721e9fdd6a3c52fcd953dfa35af846d5.tar.gz
+sha256 b5fd99ab615ca3f11ed3735ab275ca66930ce3030fad1dcaddf0abed7af90c90 no-pivot.tar.gz

deploy/iso/minikube-iso/package/crio-bin/crio-bin.mk

@@ -0,0 +1,58 @@
+################################################################################
+#
+# cri-o
+#
+################################################################################
+
+#CRIO_BIN_VERSION = 5e3c53c1721e9fdd6a3c52fcd953dfa35af846d5
+CRIO_BIN_VERSION = no-pivot
+#CRIO_BIN_SITE = https://github.com/kubernetes-incubator/cri-o/archive
+CRIO_BIN_SITE = https://github.com/vbatts/cri-o/archive
+CRIO_BIN_SOURCE = $(CRIO_BIN_VERSION).tar.gz
+CRIO_BIN_DEPENDENCIES = libgpgme
+CRIO_BIN_GOPATH = $(@D)/_output
+
+define CRIO_BIN_USERS
+	- -1 crio-admin -1 - - - - -
+	- -1 crio       -1 - - - - -
+endef
+
+define CRIO_BIN_CONFIGURE_CMDS
+	mkdir -p $(CRIO_BIN_GOPATH)/src/github.com/kubernetes-incubator
+	ln -sf $(@D) $(CRIO_BIN_GOPATH)/src/github.com/kubernetes-incubator/cri-o
+endef
+
+define CRIO_BIN_BUILD_CMDS
+	GOPATH=$(CRIO_BIN_GOPATH) $(MAKE) PREFIX=/usr BUILDTAGS="containers_image_ostree_stub" $(TARGET_CONFIGURE_OPTS) -C $(@D)
+endef
+
+define CRIO_BIN_INSTALL_TARGET_CMDS
+	mkdir -p $(TARGET_DIR)/usr/share/containers/oci/hooks.d
+	mkdir -p $(TARGET_DIR)/etc/containers/oci/hooks.d
+	mkdir -p $(TARGET_DIR)/etc/sysconfig
+	mkdir -p $(TARGET_DIR)/etc/crio
+
+	GOPATH=$(CRIO_BIN_GOPATH) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) install DESTDIR=$(TARGET_DIR) PREFIX=$(TARGET_DIR)/usr
+	$(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) install.config DESTDIR=$(TARGET_DIR) PREFIX=$(TARGET_DIR)/usr
+	$(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) install.completions DESTDIR=$(TARGET_DIR) PREFIX=$(TARGET_DIR)/usr
+
+	$(INSTALL) -Dm755 \
+		$(BR2_EXTERNAL_MINIKUBE_PATH)/package/crio-bin/crio.conf \
+		$(TARGET_DIR)/etc/crio/crio.conf
+	$(INSTALL) -Dm755 \
+		$(BR2_EXTERNAL_MINIKUBE_PATH)/package/crio-bin/policy.json \
+		$(TARGET_DIR)/etc/containers/policy.json
+	echo 'CRIO_OPTIONS="--storage-driver=overlay2 --debug"' > $(TARGET_DIR)/etc/sysconfig/crio
+
+endef
+
+define CRIO_BIN_INSTALL_INIT_SYSTEMD
+	$(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) install.systemd DESTDIR=$(TARGET_DIR) PREFIX=$(TARGET_DIR)/usr
+	$(INSTALL) -Dm755 \
+		$(BR2_EXTERNAL_MINIKUBE_PATH)/package/crio-bin/crio.service \
+		$(TARGET_DIR)/usr/lib/systemd/system/crio.service
+	$(call link-service,crio.service)
+	$(call link-service,crio-shutdown.service)
+endef
+
+$(eval $(generic-package))

deploy/iso/minikube-iso/package/crio-bin/crio.conf

@@ -0,0 +1,149 @@
+
+# The "crio" table contains all of the server options.
+[crio]
+
+# root is a path to the "root directory". CRIO stores all of its data,
+# including container images, in this directory.
+root = "/var/lib/containers/storage"
+
+# run is a path to the "run directory". CRIO stores all of its state
+# in this directory.
+runroot = "/var/run/containers/storage"
+
+# storage_driver select which storage driver is used to manage storage
+# of images and containers.
+storage_driver = ""
+
+# storage_option is used to pass an option to the storage driver.
+storage_option = [
+]
+
+# The "crio.api" table contains settings for the kubelet/gRPC
+# interface (which is also used by crioctl).
+[crio.api]
+
+# listen is the path to the AF_LOCAL socket on which crio will listen.
+listen = "/var/run/crio.sock"
+
+# stream_address is the IP address on which the stream server will listen
+stream_address = ""
+
+# stream_port is the port on which the stream server will listen
+stream_port = "10010"
+
+# file_locking is whether file-based locking will be used instead of
+# in-memory locking
+file_locking = true
+
+# The "crio.runtime" table contains settings pertaining to the OCI
+# runtime used and options for how to set up and manage the OCI runtime.
+[crio.runtime]
+
+# runtime is the OCI compatible runtime used for trusted container workloads.
+# This is a mandatory setting as this runtime will be the default one
+# and will also be used for untrusted container workloads if
+# runtime_untrusted_workload is not set.
+runtime = "/usr/bin/runc"
+
+# runtime_untrusted_workload is the OCI compatible runtime used for untrusted
+# container workloads. This is an optional setting, except if
+# default_container_trust is set to "untrusted".
+runtime_untrusted_workload = ""
+
+# default_workload_trust is the default level of trust crio puts in container
+# workloads. It can either be "trusted" or "untrusted", and the default
+# is "trusted".
+# Containers can be run through different container runtimes, depending on
+# the trust hints we receive from kubelet:
+# - If kubelet tags a container workload as untrusted, crio will try first to
+# run it through the untrusted container workload runtime. If it is not set,
+# crio will use the trusted runtime.
+# - If kubelet does not provide any information about the container workload trust
+# level, the selected runtime will depend on the default_container_trust setting.
+# If it is set to "untrusted", then all containers except for the host privileged
+# ones, will be run by the runtime_untrusted_workload runtime. Host privileged
+# containers are by definition trusted and will always use the trusted container
+# runtime. If default_container_trust is set to "trusted", crio will use the trusted
+# container runtime for all containers.
+default_workload_trust = "trusted"
+
+# no_pivot instructs the runtime to not use pivot_root, but instead use MS_MOVE
+no_pivot = true
+
+# conmon is the path to conmon binary, used for managing the runtime.
+conmon = "/usr/libexec/crio/conmon"
+
+# conmon_env is the environment variable list for conmon process,
+# used for passing necessary environment variable to conmon or runtime.
+conmon_env = [
+	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+]
+
+# selinux indicates whether or not SELinux will be used for pod
+# separation on the host. If you enable this flag, SELinux must be running
+# on the host.
+selinux = false
+
+# seccomp_profile is the seccomp json profile path which is used as the
+# default for the runtime.
+seccomp_profile = "/etc/crio/seccomp.json"
+
+# apparmor_profile is the apparmor profile name which is used as the
+# default for the runtime.
+apparmor_profile = "crio-default"
+
+# cgroup_manager is the cgroup management implementation to be used
+# for the runtime.
+cgroup_manager = "cgroupfs"
+
+# hooks_dir_path is the oci hooks directory for automatically executed hooks
+hooks_dir_path = "/usr/share/containers/oci/hooks.d"
+
+# pids_limit is the number of processes allowed in a container
+pids_limit = 1024
+
+# The "crio.image" table contains settings pertaining to the
+# management of OCI images.
+[crio.image]
+
+# default_transport is the prefix we try prepending to an image name if the
+# image name as we receive it can't be parsed as a valid source reference
+default_transport = "docker://"
+
+# pause_image is the image which we use to instantiate infra containers.
+pause_image = "kubernetes/pause"
+
+# pause_command is the command to run in a pause_image to have a container just
+# sit there.  If the image contains the necessary information, this value need
+# not be specified.
+pause_command = "/pause"
+
+# signature_policy is the name of the file which decides what sort of policy we
+# use when deciding whether or not to trust an image that we've pulled.
+# Outside of testing situations, it is strongly advised that this be left
+# unspecified so that the default system-wide policy will be used.
+signature_policy = ""
+
+# image_volumes controls how image volumes are handled.
+# The valid values are mkdir and ignore.
+image_volumes = "mkdir"
+
+# insecure_registries is used to skip TLS verification when pulling images.
+insecure_registries = [
+]
+
+# registries is used to specify a comma separated list of registries to be used
+# when pulling an unqualified image (e.g. fedora:rawhide).
+registries = [
+]
+
+# The "crio.network" table contains settings pertaining to the
+# management of CNI plugins.
+[crio.network]
+
+# network_dir is is where CNI network configuration
+# files are stored.
+network_dir = "/etc/cni/net.d/"
+
+# plugin_dir is is where CNI plugin binaries are stored.
+plugin_dir = "/opt/cni/bin/"

deploy/iso/minikube-iso/package/crio-bin/crio.service

@@ -0,0 +1,26 @@
+[Unit]
+Description=Open Container Initiative Daemon
+Documentation=https://github.com/kubernetes-incubator/cri-o
+After=network-online.target minikube-automount.service
+Requires=minikube-automount.service
+
+[Service]
+Type=notify
+EnvironmentFile=-/etc/sysconfig/crio
+EnvironmentFile=/var/run/minikube/env
+Environment=GOTRACEBACK=crash
+ExecStartPre=/bin/mkdir -p ${PERSISTENT_DIR}/var/lib/containers
+ExecStart=/usr/bin/crio \
+          $CRIO_OPTIONS \
+	  --root ${PERSISTENT_DIR}/var/lib/containers
+ExecReload=/bin/kill -s HUP $MAINPID
+TasksMax=8192
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+OOMScoreAdjust=-999
+TimeoutStartSec=0
+Restart=on-abnormal
+
+[Install]
+WantedBy=multi-user.target

deploy/iso/minikube-iso/package/crio-bin/policy.json

@@ -0,0 +1,7 @@
+{
+  "default": [
+    {
+      "type": "insecureAcceptAnything"
+    }
+  ]
+}

deploy/iso/minikube-iso/package/rkt-bin/rkt-bin.mk

@@ -52,18 +52,21 @@ define RKT_BIN_INSTALL_INIT_SYSTEMD
 		$(@D)/init/systemd/tmpfiles.d/rkt.conf \
 		$(TARGET_DIR)/usr/lib/tmpfiles.d/rkt.conf
 
-	$(call install-service,rkt-api.service)
-	$(call install-service,rkt-gc.timer)
-	$(call install-service,rkt-gc.service)
-	$(call install-service,rkt-metadata.socket)
-	$(call install-service,rkt-metadata.service)
+	$(call rkt-install-service,rkt-api.service)
+	$(call rkt-install-service,rkt-gc.timer)
+	$(call rkt-install-service,rkt-gc.service)
+	$(call rkt-install-service,rkt-metadata.socket)
+	$(call rkt-install-service,rkt-metadata.service)
 endef
 
-define install-service
+define rkt-install-service
 	$(INSTALL) -D -m 644 \
 		$(@D)/init/systemd/$(1) \
 		$(TARGET_DIR)/usr/lib/systemd/system/$(1)
+	$(call link-service,$(1))
+endef
 
+define link-service
 	ln -fs /usr/lib/systemd/system/$(1) \
 		$(TARGET_DIR)/etc/systemd/system/multi-user.target.wants/$(1)
 endef

deploy/iso/minikube-iso/package/runc-master/Config.in

@@ -0,0 +1,17 @@
+config BR2_PACKAGE_RUNC_MASTER
+	bool "runc-master"
+	depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS
+	depends on BR2_PACKAGE_HOST_GO_CGO_LINKING_SUPPORTS
+	depends on BR2_TOOLCHAIN_HAS_THREADS
+	help
+	  runC is a CLI tool for spawning and running containers
+	  according to the OCP specification.
+
+	  This is just a newer build of runc than the buildroot version.
+
+	  https://github.com/opencontainers/runc
+
+comment "runc needs a toolchain w/ threads"
+	depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS && \
+		BR2_PACKAGE_HOST_GO_CGO_LINKING_SUPPORTS
+	depends on !BR2_TOOLCHAIN_HAS_THREADS

deploy/iso/minikube-iso/package/runc-master/runc-master.hash

@@ -0,0 +1,2 @@
+# Locally computed
+sha256 e9ad8aa5590f65a23326b7e9944d8b9881fa002ccb4a8e2cd40712a89a40ee45  runc-master-593914b8bd5448a93f7c3e4902a03408b6d5c0ce.tar.gz