Allow MicroShift to join new worker nodes

.gitignore

@@ -5,5 +5,4 @@ bin
 .idea/
 .vscode/
 .vagrant/
-Vagrantfile
 sshfile

Makefile

@@ -65,7 +65,8 @@ debug:
 	@echo SOURCE_GIT_TAG:"$(SOURCE_GIT_TAG)"
 
 # These tags make sure we can statically link and avoid shared dependencies
-GO_BUILD_FLAGS :=-tags 'include_gcs include_oss containers_image_openpgp gssapi providerless netgo osusergo'
+# netcgo : use system resolver for DNS instead of the netgo implementation which does not include mDNS
+GO_BUILD_FLAGS :=-tags 'include_gcs include_oss containers_image_openpgp gssapi providerless netcgo osusergo'
 
 # targets "all:" and "build:" defined in vendor/github.com/openshift/build-machinery-go/make/targets/golang/build.mk
 microshift: build-containerized-cross-build-linux-amd64

assets/rbac/0000_10_bootstrap-crb-approver.yaml

@@ -0,0 +1,14 @@
+# Approve all CSRs for the group "system:bootstrappers"
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: auto-approve-csrs-for-group
+subjects:
+- kind: Group
+  name: system:bootstrappers
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
+  apiGroup: rbac.authorization.k8s.io
+

assets/rbac/0000_10_bootstrap-crb-creator.yaml

@@ -0,0 +1,14 @@
+# enable bootstrapping nodes to create CSR
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: create-csrs-for-bootstrapping
+subjects:
+- kind: Group
+  name: system:bootstrappers
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: system:node-bootstrapper
+  apiGroup: rbac.authorization.k8s.io
+

assets/rbac/0000_10_bootstrap-crb-renewal.yaml

@@ -0,0 +1,13 @@
+# approve renewal CSRs for the group "system:nodes"
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: auto-approve-renewals-for-nodes
+subjects:
+- kind: Group
+  name: system:nodes
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
+  apiGroup: rbac.authorization.k8s.io

pkg/assets/rbac/bindata.go

@@ -3,6 +3,9 @@
 // assets/rbac/0000_00_flannel-clusterrole.yaml
 // assets/rbac/0000_00_flannel-clusterrolebinding.yaml
 // assets/rbac/0000_00_podsecuritypolicy-flannel.yaml
+// assets/rbac/0000_10_bootstrap-crb-approver.yaml
+// assets/rbac/0000_10_bootstrap-crb-creator.yaml
+// assets/rbac/0000_10_bootstrap-crb-renewal.yaml
 // assets/rbac/0000_60_service-ca_00_clusterrole.yaml
 // assets/rbac/0000_60_service-ca_00_clusterrolebinding.yaml
 // assets/rbac/0000_60_service-ca_00_role.yaml
@@ -200,6 +203,96 @@ func assetsRbac0000_00_podsecuritypolicyFlannelYaml() (*asset, error) {
 	return a, nil
 }
 
+var _assetsRbac0000_10_bootstrapCrbApproverYaml = []byte(`# Approve all CSRs for the group "system:bootstrappers"
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: auto-approve-csrs-for-group
+subjects:
+- kind: Group
+  name: system:bootstrappers
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
+  apiGroup: rbac.authorization.k8s.io
+  `)
+
+func assetsRbac0000_10_bootstrapCrbApproverYamlBytes() ([]byte, error) {
+	return _assetsRbac0000_10_bootstrapCrbApproverYaml, nil
+}
+
+func assetsRbac0000_10_bootstrapCrbApproverYaml() (*asset, error) {
+	bytes, err := assetsRbac0000_10_bootstrapCrbApproverYamlBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "assets/rbac/0000_10_bootstrap-crb-approver.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
+	a := &asset{bytes: bytes, info: info}
+	return a, nil
+}
+
+var _assetsRbac0000_10_bootstrapCrbCreatorYaml = []byte(`# enable bootstrapping nodes to create CSR
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: create-csrs-for-bootstrapping
+subjects:
+- kind: Group
+  name: system:bootstrappers
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: system:node-bootstrapper
+  apiGroup: rbac.authorization.k8s.io
+  `)
+
+func assetsRbac0000_10_bootstrapCrbCreatorYamlBytes() ([]byte, error) {
+	return _assetsRbac0000_10_bootstrapCrbCreatorYaml, nil
+}
+
+func assetsRbac0000_10_bootstrapCrbCreatorYaml() (*asset, error) {
+	bytes, err := assetsRbac0000_10_bootstrapCrbCreatorYamlBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "assets/rbac/0000_10_bootstrap-crb-creator.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
+	a := &asset{bytes: bytes, info: info}
+	return a, nil
+}
+
+var _assetsRbac0000_10_bootstrapCrbRenewalYaml = []byte(`# approve renewal CSRs for the group "system:nodes"
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: auto-approve-renewals-for-nodes
+subjects:
+- kind: Group
+  name: system:nodes
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
+  apiGroup: rbac.authorization.k8s.io
+`)
+
+func assetsRbac0000_10_bootstrapCrbRenewalYamlBytes() ([]byte, error) {
+	return _assetsRbac0000_10_bootstrapCrbRenewalYaml, nil
+}
+
+func assetsRbac0000_10_bootstrapCrbRenewalYaml() (*asset, error) {
+	bytes, err := assetsRbac0000_10_bootstrapCrbRenewalYamlBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "assets/rbac/0000_10_bootstrap-crb-renewal.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
+	a := &asset{bytes: bytes, info: info}
+	return a, nil
+}
+
 var _assetsRbac0000_60_serviceCa_00_clusterroleYaml = []byte(`apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -715,6 +808,9 @@ var _bindata = map[string]func() (*asset, error){
 	"assets/rbac/0000_00_flannel-clusterrole.yaml":                     assetsRbac0000_00_flannelClusterroleYaml,
 	"assets/rbac/0000_00_flannel-clusterrolebinding.yaml":              assetsRbac0000_00_flannelClusterrolebindingYaml,
 	"assets/rbac/0000_00_podsecuritypolicy-flannel.yaml":               assetsRbac0000_00_podsecuritypolicyFlannelYaml,
+	"assets/rbac/0000_10_bootstrap-crb-approver.yaml":                  assetsRbac0000_10_bootstrapCrbApproverYaml,
+	"assets/rbac/0000_10_bootstrap-crb-creator.yaml":                   assetsRbac0000_10_bootstrapCrbCreatorYaml,
+	"assets/rbac/0000_10_bootstrap-crb-renewal.yaml":                   assetsRbac0000_10_bootstrapCrbRenewalYaml,
 	"assets/rbac/0000_60_service-ca_00_clusterrole.yaml":               assetsRbac0000_60_serviceCa_00_clusterroleYaml,
 	"assets/rbac/0000_60_service-ca_00_clusterrolebinding.yaml":        assetsRbac0000_60_serviceCa_00_clusterrolebindingYaml,
 	"assets/rbac/0000_60_service-ca_00_role.yaml":                      assetsRbac0000_60_serviceCa_00_roleYaml,
@@ -773,6 +869,9 @@ var _bintree = &bintree{nil, map[string]*bintree{
 			"0000_00_flannel-clusterrole.yaml":                     {assetsRbac0000_00_flannelClusterroleYaml, map[string]*bintree{}},
 			"0000_00_flannel-clusterrolebinding.yaml":              {assetsRbac0000_00_flannelClusterrolebindingYaml, map[string]*bintree{}},
 			"0000_00_podsecuritypolicy-flannel.yaml":               {assetsRbac0000_00_podsecuritypolicyFlannelYaml, map[string]*bintree{}},
+			"0000_10_bootstrap-crb-approver.yaml":                  {assetsRbac0000_10_bootstrapCrbApproverYaml, map[string]*bintree{}},
+			"0000_10_bootstrap-crb-creator.yaml":                   {assetsRbac0000_10_bootstrapCrbCreatorYaml, map[string]*bintree{}},
+			"0000_10_bootstrap-crb-renewal.yaml":                   {assetsRbac0000_10_bootstrapCrbRenewalYaml, map[string]*bintree{}},
 			"0000_60_service-ca_00_clusterrole.yaml":               {assetsRbac0000_60_serviceCa_00_clusterroleYaml, map[string]*bintree{}},
 			"0000_60_service-ca_00_clusterrolebinding.yaml":        {assetsRbac0000_60_serviceCa_00_clusterrolebindingYaml, map[string]*bintree{}},
 			"0000_60_service-ca_00_role.yaml":                      {assetsRbac0000_60_serviceCa_00_roleYaml, map[string]*bintree{}},

pkg/bootstrap/bootstrap.go

@@ -0,0 +1,24 @@
+package bootstrap
+
+import (
+	"github.com/openshift/microshift/pkg/assets"
+	"github.com/openshift/microshift/pkg/config"
+	"github.com/sirupsen/logrus"
+)
+
+func ApplyBootstrapClusterRoleBindings(cfg *config.MicroshiftConfig, kubeconfigPath string) error {
+	var (
+		clusterRoleBinding = []string{
+			"assets/rbac/0000_10_bootstrap-crb-creator.yaml",
+			"assets/rbac/0000_10_bootstrap-crb-approver.yaml",
+			"assets/rbac/0000_10_bootstrap-crb-renewal.yaml",
+		}
+	)
+
+	if err := assets.ApplyClusterRoleBindings(clusterRoleBinding, kubeconfigPath); err != nil {
+		logrus.Warningf("failed to apply clusterRolebinding %v: %v", clusterRoleBinding, err)
+		return err
+	}
+
+	return nil
+}

pkg/bootstrap/token-manager.go

@@ -0,0 +1,40 @@
+package bootstrap
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/openshift/microshift/pkg/config"
+	"github.com/openshift/microshift/pkg/util"
+)
+
+type TokenManager struct {
+	path string
+	cfg  *config.MicroshiftConfig
+}
+
+func NewTokenManager(cfg *config.MicroshiftConfig) *TokenManager {
+	return &TokenManager{
+		path: filepath.Join(cfg.DataDir, "resources", "microshift-bootstrap-token"),
+		cfg:  cfg,
+	}
+}
+
+func (s *TokenManager) Name() string           { return "token-manager" }
+func (s *TokenManager) Dependencies() []string { return []string{} }
+
+func (s *TokenManager) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error {
+	defer close(stopped)
+	defer close(ready)
+
+	CreateTokenFile(s.path)
+	_, err := os.Stat(s.cfg.DataDir + "/resources/kubelet/bootstrap-kubeconfig")
+	if os.IsNotExist(err) {
+		if err := util.BootstrapKubeconfig(GetToken(s.path), s.cfg.DataDir+"/resources/kubelet/bootstrap-kubeconfig", "system:bootstrappers", []string{"system:bootstrappers"}, s.cfg.Cluster.URL); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/bootstrap/token.go

@@ -0,0 +1,47 @@
+package bootstrap
+
+import (
+	"encoding/hex"
+	"math/rand"
+	"os"
+	"strings"
+	"time"
+
+	"k8s.io/klog/v2"
+)
+
+func CreateTokenFile(path string) {
+	err := os.Remove(path)
+	if err != nil {
+		klog.ErrorS(err, "Token file does not exist")
+	}
+
+	token := randString(16)
+	token = token + ",kubelet-bootstrap,10001,\"system:bootstrappers\""
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		klog.ErrorS(err, "Token file cannot be created")
+	}
+	defer f.Close()
+	f.Write([]byte(token))
+
+}
+
+func GetToken(path string) string {
+	f, err := os.ReadFile(path)
+	if err != nil {
+		klog.ErrorS(err, "Token file cannot be opened")
+	}
+	token := f[:strings.IndexByte(string(f), ',')]
+
+	return string(token)
+}
+
+func randString(length int) string {
+	b := make([]byte, length)
+	rand.Seed(time.Now().UnixNano())
+	if _, err := rand.Read(b); err != nil {
+		return ""
+	}
+	return hex.EncodeToString(b)
+}

pkg/cmd/run.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/coreos/go-systemd/daemon"
+	"github.com/openshift/microshift/pkg/bootstrap"
 	"github.com/openshift/microshift/pkg/config"
 	"github.com/openshift/microshift/pkg/controllers"
 	"github.com/openshift/microshift/pkg/kustomize"
@@ -63,11 +64,14 @@ func RunMicroshift(cfg *config.MicroshiftConfig, flags *pflag.FlagSet) error {
 
 	// TODO: change to only initialize what is strictly necessary for the selected role(s)
 	if _, err := os.Stat(filepath.Join(cfg.DataDir, "certs")); errors.Is(err, os.ErrNotExist) {
-		initAll(cfg)
+		if config.StringInList("controlplane", cfg.Roles) {
+			initAll(cfg)
+		}
 	}
 
 	m := servicemanager.NewServiceManager()
 	if config.StringInList("controlplane", cfg.Roles) {
+		util.Must(m.AddService(bootstrap.NewTokenManager(cfg)))
 		util.Must(m.AddService(controllers.NewEtcd(cfg)))
 		util.Must(m.AddService(controllers.NewKubeAPIServer(cfg)))
 		util.Must(m.AddService(controllers.NewKubeScheduler(cfg)))

pkg/components/components.go

@@ -1,6 +1,7 @@
 package components
 
 import (
+	"github.com/openshift/microshift/pkg/bootstrap"
 	"github.com/openshift/microshift/pkg/config"
 	"github.com/sirupsen/logrus"
 )
@@ -28,5 +29,9 @@ func StartComponents(cfg *config.MicroshiftConfig) error {
 		logrus.Warningf("failed to start Flannel: %v", err)
 		return err
 	}
+	if err := bootstrap.ApplyBootstrapClusterRoleBindings(cfg, cfg.DataDir+"/resources/kubeadmin/kubeconfig"); err != nil {
+		logrus.Warningf("failed to start Flannel: %v", err)
+		return err
+	}
 	return nil
 }

pkg/config/config.go

@@ -9,7 +9,7 @@ import (
 	"strconv"
 
 	"github.com/kelseyhightower/envconfig"
-	homedir "github.com/mitchellh/go-homedir"
+	"github.com/mitchellh/go-homedir"
 	"github.com/openshift/microshift/pkg/util"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
@@ -86,7 +86,7 @@ func NewMicroshiftConfig() *MicroshiftConfig {
 		NodeName:        nodeName,
 		NodeIP:          nodeIP,
 		Cluster: ClusterConfig{
-			URL:         "https://127.0.0.1:6443",
+			URL:         util.NodeURL(nodeIP, nodeName),
 			ClusterCIDR: "10.42.0.0/16",
 			ServiceCIDR: "10.43.0.0/16",
 			DNS:         "10.43.0.10",

pkg/controllers/kube-apiserver.go

@@ -55,7 +55,7 @@ func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer {
 }
 
 func (s *KubeAPIServer) Name() string           { return "kube-apiserver" }
-func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd"} }
+func (s *KubeAPIServer) Dependencies() []string { return []string{"token-manager", "etcd"} }
 
 func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) {
 	caCertFile := filepath.Join(cfg.DataDir, "certs", "ca-bundle", "ca-bundle.crt")
@@ -92,6 +92,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) {
 		"--client-ca-file=" + caCertFile,
 		"--enable-admission-plugins=NodeRestriction",
 		"--enable-aggregator-routing=true",
+		"--enable-bootstrap-token-auth",
 		"--etcd-cafile=" + caCertFile,
 		"--etcd-certfile=" + cfg.DataDir + "/resources/kube-apiserver/secrets/etcd-client/tls.crt",
 		"--etcd-keyfile=" + cfg.DataDir + "/resources/kube-apiserver/secrets/etcd-client/tls.key",
@@ -114,6 +115,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) {
 		"--storage-backend=etcd3",
 		"--tls-cert-file=" + cfg.DataDir + "/certs/kube-apiserver/secrets/service-network-serving-certkey/tls.crt",
 		"--tls-private-key-file=" + cfg.DataDir + "/certs/kube-apiserver/secrets/service-network-serving-certkey/tls.key",
+		"--token-auth-file=" + cfg.DataDir + "/resources/microshift-bootstrap-token",
 		"--cors-allowed-origins=/127.0.0.1(:[0-9]+)?$,/localhost(:[0-9]+)?$",
 		"--logtostderr=" + strconv.FormatBool(cfg.LogDir == "" || cfg.LogAlsotostderr),
 		"--alsologtostderr=" + strconv.FormatBool(cfg.LogAlsotostderr),

pkg/controllers/kube-controller-manager.go

@@ -62,6 +62,7 @@ func (s *KubeControllerManager) configure(cfg *config.MicroshiftConfig) {
 		"--service-account-private-key-file=" + cfg.DataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.key",
 		"--allocate-node-cidrs=true",
 		"--cluster-cidr=" + cfg.Cluster.ClusterCIDR,
+		"--controllers=*,tokencleaner,bootstrapsigner",
 		"--authorization-kubeconfig=" + kubeconfig,
 		"--authentication-kubeconfig=" + kubeconfig,
 		"--root-ca-file=" + caCertFile,