Skip to content

Commit

Permalink
kinder: support the --experimental-patches flag for kubeadm
Browse files Browse the repository at this point in the history 
- Add the flag "--patches" for "kinder do"
- Bind the flag to init, join, upgrade.
- Support only Nodes that are >= 1.19.
- Use the same directory constant for kustomize / patches "PatchesDir".
  • Loading branch information
@neolit123
neolit123 committed Jun 14, 2020
1 parent 76f8bde commit 98e1626
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 34 deletions.
7 changes: 7 additions & 0 deletions kinder/cmd/kinder/do/do.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type flagpole struct {
DryRun bool
VLevel int
KustomizeDir string
PatchesDir string
Wait time.Duration
}

Expand Down Expand Up @@ -114,6 +115,11 @@ func NewCommand() *cobra.Command {
"kustomize-dir", "k", flags.KustomizeDir,
"the kustomize folder to be used for init,join and upgrade",
)
cmd.Flags().StringVar(
&flags.PatchesDir,
"patches", flags.PatchesDir,
"the patches directory to be used for init, join and upgrade",
)
return cmd
}

Expand Down Expand Up @@ -161,6 +167,7 @@ func runE(flags *flagpole, cmd *cobra.Command, args []string) (err error) {
actions.UpgradeVersion(upgradeVersion),
actions.VLevel(flags.VLevel),
actions.KustomizeDir(flags.KustomizeDir),
actions.PatchesDir(flags.PatchesDir),
)
if err != nil {
return errors.Wrapf(err, "failed to exec action %s", action)
Expand Down
14 changes: 11 additions & 3 deletions kinder/pkg/cluster/manager/actions/actions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,13 @@ var actionRegistry = map[string]func(*status.Cluster, *RunOptions) error{
return KubeadmConfig(c, flags.kubeDNS, flags.automaticCopyCerts, flags.discoveryMode, c.K8sNodes().EligibleForActions()...)
},
"kubeadm-init": func(c *status.Cluster, flags *RunOptions) error {
return KubeadmInit(c, flags.usePhases, flags.kubeDNS, flags.automaticCopyCerts, flags.kustomizeDir, flags.wait, flags.vLevel)
return KubeadmInit(c, flags.usePhases, flags.kubeDNS, flags.automaticCopyCerts, flags.kustomizeDir, flags.patchesDir, flags.wait, flags.vLevel)
},
"kubeadm-join": func(c *status.Cluster, flags *RunOptions) error {
return KubeadmJoin(c, flags.usePhases, flags.automaticCopyCerts, flags.discoveryMode, flags.kustomizeDir, flags.wait, flags.vLevel)
return KubeadmJoin(c, flags.usePhases, flags.automaticCopyCerts, flags.discoveryMode, flags.kustomizeDir, flags.patchesDir, flags.wait, flags.vLevel)
},
"kubeadm-upgrade": func(c *status.Cluster, flags *RunOptions) error {
return KubeadmUpgrade(c, flags.upgradeVersion, flags.kustomizeDir, flags.wait, flags.vLevel)
return KubeadmUpgrade(c, flags.upgradeVersion, flags.kustomizeDir, flags.patchesDir, flags.wait, flags.vLevel)
},
"kubeadm-reset": func(c *status.Cluster, flags *RunOptions) error {
return KubeadmReset(c, flags.vLevel)
Expand Down Expand Up @@ -135,6 +135,13 @@ func KustomizeDir(kustomizeDir string) Option {
}
}

// PatchesDir option sets the patches dir for the kubeadm commands
func PatchesDir(patchesDir string) Option {
return func(r *RunOptions) {
r.patchesDir = patchesDir
}
}

// RunOptions holds options supplied to actions.Run
type RunOptions struct {
kubeDNS bool
Expand All @@ -145,6 +152,7 @@ type RunOptions struct {
upgradeVersion *K8sVersion.Version
vLevel int
kustomizeDir string
patchesDir string
}

// DiscoveryMode defines discovery mode supported by kubeadm join
Expand Down
40 changes: 28 additions & 12 deletions kinder/pkg/cluster/manager/actions/kubeadm-init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import (

// KubeadmInit executes the kubeadm init workflow including also post init task
// like installing the CNI network plugin
func KubeadmInit(c *status.Cluster, usePhases, kubeDNS, automaticCopyCerts bool, kustomizeDir string, wait time.Duration, vLevel int) (err error) {
func KubeadmInit(c *status.Cluster, usePhases, kubeDNS, automaticCopyCerts bool, kustomizeDir, patchesDir string, wait time.Duration, vLevel int) (err error) {
cp1 := c.BootstrapControlPlane()

// fail fast if required to use kustomize and kubeadm less than v1.16
Expand All @@ -52,6 +52,16 @@ func KubeadmInit(c *status.Cluster, usePhases, kubeDNS, automaticCopyCerts bool,
}
}

// if patcheDir is defined, copy the patches to the node
if patchesDir != "" {
if cp1.MustKubeadmVersion().LessThan(constants.V1_19) {
return errors.New("--patches can't be used with kubeadm older than v1.19")
}
if err := copyPatchesToNode(cp1, patchesDir); err != nil {
return err
}
}

// checks pre-loaded images available on the node (this will report missing images, if any)
kubeVersion, err := cp1.KubeVersion()
if err != nil {
Expand All @@ -74,7 +84,7 @@ func KubeadmInit(c *status.Cluster, usePhases, kubeDNS, automaticCopyCerts bool,

// execs the kubeadm init workflow
if usePhases {
err = kubeadmInitWithPhases(cp1, automaticCopyCerts, kustomizeDir, vLevel)
err = kubeadmInitWithPhases(cp1, automaticCopyCerts, kustomizeDir, patchesDir, vLevel)
} else {
err = kubeadmInit(cp1, automaticCopyCerts, kustomizeDir, vLevel)
}
Expand Down Expand Up @@ -112,7 +122,7 @@ func kubeadmInit(cp1 *status.Node, automaticCopyCerts bool, kustomizeDir string,
}
}
if kustomizeDir != "" {
initArgs = append(initArgs, "-k", constants.KustomizeDir)
initArgs = append(initArgs, "-k", constants.PatchesDir)
}

if err := cp1.Command(
Expand All @@ -124,7 +134,7 @@ func kubeadmInit(cp1 *status.Node, automaticCopyCerts bool, kustomizeDir string,
return nil
}

func kubeadmInitWithPhases(cp1 *status.Node, automaticCopyCerts bool, kustomizeDir string, vLevel int) error {
func kubeadmInitWithPhases(cp1 *status.Node, automaticCopyCerts bool, kustomizeDir, patchesDir string, vLevel int) error {
if err := cp1.Command(
"kubeadm", "init", "phase", "preflight", fmt.Sprintf("--config=%s", constants.KubeadmConfigPath), fmt.Sprintf("--v=%d", vLevel),
constants.KubeadmIgnorePreflightErrorsFlag,
Expand Down Expand Up @@ -154,7 +164,10 @@ func kubeadmInitWithPhases(cp1 *status.Node, automaticCopyCerts bool, kustomizeD
"init", "phase", "control-plane", "all", fmt.Sprintf("--config=%s", constants.KubeadmConfigPath), fmt.Sprintf("--v=%d", vLevel),
}
if kustomizeDir != "" {
controlplaneArgs = append(controlplaneArgs, "-k", constants.KustomizeDir)
controlplaneArgs = append(controlplaneArgs, "-k", constants.PatchesDir)
}
if patchesDir != "" {
controlplaneArgs = append(controlplaneArgs, "--experimental-patches", constants.PatchesDir)
}
if err := cp1.Command(
"kubeadm", controlplaneArgs...,
Expand All @@ -166,7 +179,10 @@ func kubeadmInitWithPhases(cp1 *status.Node, automaticCopyCerts bool, kustomizeD
"init", "phase", "etcd", "local", fmt.Sprintf("--config=%s", constants.KubeadmConfigPath), fmt.Sprintf("--v=%d", vLevel),
}
if kustomizeDir != "" {
etcdArgs = append(etcdArgs, "-k", constants.KustomizeDir)
etcdArgs = append(etcdArgs, "-k", constants.PatchesDir)
}
if patchesDir != "" {
controlplaneArgs = append(controlplaneArgs, "--experimental-patches", constants.PatchesDir)
}
if err := cp1.Command(
"kubeadm", etcdArgs...,
Expand Down Expand Up @@ -368,14 +384,14 @@ func writeKubeConfig(c *status.Cluster, hostPort int32) error {

func copyPatchesToNode(n *status.Node, dir string) error {

n.Infof("Importing kustomize patches from %s", dir)
n.Infof("Importing patches from %s", dir)

// creates the folder tree for kustomize patches
if err := n.Command("mkdir", "-p", constants.KustomizeDir).Silent().Run(); err != nil {
return errors.Wrapf(err, "failed to create %s folder", constants.KustomizeDir)
// creates the folder tree for patches
if err := n.Command("mkdir", "-p", constants.PatchesDir).Silent().Run(); err != nil {
return errors.Wrapf(err, "failed to create %s folder", constants.PatchesDir)
}

// copies kustomize patches
// copies patches
files, err := ioutil.ReadDir(dir)
if err != nil {
log.Fatal(err)
Expand All @@ -385,7 +401,7 @@ func copyPatchesToNode(n *status.Node, dir string) error {
fmt.Printf("%s\n", file.Name())

hostPath := filepath.Join(dir, file.Name())
nodePath := filepath.Join(constants.KustomizeDir, file.Name())
nodePath := filepath.Join(constants.PatchesDir, file.Name())

if err := n.CopyTo(hostPath, nodePath); err != nil {
errors.Wrapf(err, "failed to copy from host path %q to node path %q for node %q",
Expand Down
39 changes: 29 additions & 10 deletions kinder/pkg/cluster/manager/actions/kubeadm-join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ import (

// KubeadmJoin executes the kubeadm join workflow both for control-plane nodes and
// worker nodes
func KubeadmJoin(c *status.Cluster, usePhases, automaticCopyCerts bool, discoveryMode DiscoveryMode, kustomizeDir string, wait time.Duration, vLevel int) (err error) {
if err := joinControlPlanes(c, usePhases, automaticCopyCerts, discoveryMode, kustomizeDir, wait, vLevel); err != nil {
func KubeadmJoin(c *status.Cluster, usePhases, automaticCopyCerts bool, discoveryMode DiscoveryMode, kustomizeDir, patchesDir string, wait time.Duration, vLevel int) (err error) {
if err := joinControlPlanes(c, usePhases, automaticCopyCerts, discoveryMode, kustomizeDir, patchesDir, wait, vLevel); err != nil {
return err
}

Expand All @@ -39,7 +39,7 @@ func KubeadmJoin(c *status.Cluster, usePhases, automaticCopyCerts bool, discover
return nil
}

func joinControlPlanes(c *status.Cluster, usePhases, automaticCopyCerts bool, discoveryMode DiscoveryMode, kustomizeDir string, wait time.Duration, vLevel int) (err error) {
func joinControlPlanes(c *status.Cluster, usePhases, automaticCopyCerts bool, discoveryMode DiscoveryMode, kustomizeDir, patchesDir string, wait time.Duration, vLevel int) (err error) {
cpX := []*status.Node{c.BootstrapControlPlane()}

for _, cp2 := range c.SecondaryControlPlanes().EligibleForActions() {
Expand All @@ -55,6 +55,16 @@ func joinControlPlanes(c *status.Cluster, usePhases, automaticCopyCerts bool, di
}
}

// if patcheDir is defined, copy the patches to the node
if patchesDir != "" {
if cp2.MustKubeadmVersion().LessThan(constants.V1_19) {
return errors.New("--patches can't be used with kubeadm older than v1.19")
}
if err := copyPatchesToNode(cp2, patchesDir); err != nil {
return err
}
}

// if not automatic copy certs, simulate manual copy
if !automaticCopyCerts {
if err := copyCertificatesToNode(c, cp2); err != nil {
Expand All @@ -80,9 +90,9 @@ func joinControlPlanes(c *status.Cluster, usePhases, automaticCopyCerts bool, di

// executes the kubeadm join control-plane workflow
if usePhases {
err = kubeadmJoinControlPlaneWithPhases(cp2, automaticCopyCerts, kustomizeDir, vLevel)
err = kubeadmJoinControlPlaneWithPhases(cp2, automaticCopyCerts, kustomizeDir, patchesDir, vLevel)
} else {
err = kubeadmJoinControlPlane(cp2, automaticCopyCerts, kustomizeDir, vLevel)
err = kubeadmJoinControlPlane(cp2, automaticCopyCerts, kustomizeDir, patchesDir, vLevel)
}
if err != nil {
return err
Expand All @@ -101,7 +111,7 @@ func joinControlPlanes(c *status.Cluster, usePhases, automaticCopyCerts bool, di
return nil
}

func kubeadmJoinControlPlane(cp *status.Node, automaticCopyCerts bool, kustomizeDir string, vLevel int) (err error) {
func kubeadmJoinControlPlane(cp *status.Node, automaticCopyCerts bool, kustomizeDir, patchesDir string, vLevel int) (err error) {
joinArgs := []string{
"join",
fmt.Sprintf("--config=%s", constants.KubeadmConfigPath),
Expand All @@ -117,7 +127,10 @@ func kubeadmJoinControlPlane(cp *status.Node, automaticCopyCerts bool, kustomize
}
}
if kustomizeDir != "" {
joinArgs = append(joinArgs, "-k", constants.KustomizeDir)
joinArgs = append(joinArgs, "-k", constants.PatchesDir)
}
if patchesDir != "" {
joinArgs = append(joinArgs, "--experimental-patches", constants.PatchesDir)
}

if err := cp.Command(
Expand All @@ -129,7 +142,7 @@ func kubeadmJoinControlPlane(cp *status.Node, automaticCopyCerts bool, kustomize
return nil
}

func kubeadmJoinControlPlaneWithPhases(cp *status.Node, automaticCopyCerts bool, kustomizeDir string, vLevel int) (err error) {
func kubeadmJoinControlPlaneWithPhases(cp *status.Node, automaticCopyCerts bool, kustomizeDir, patchesDir string, vLevel int) (err error) {
// kubeadm join phase preflight
preflightArgs := []string{
"join", "phase", "preflight",
Expand Down Expand Up @@ -167,7 +180,10 @@ func kubeadmJoinControlPlaneWithPhases(cp *status.Node, automaticCopyCerts bool,
}
}
if kustomizeDir != "" {
prepareArgs = append(prepareArgs, "-k", constants.KustomizeDir)
prepareArgs = append(prepareArgs, "-k", constants.PatchesDir)
}
if patchesDir != "" {
prepareArgs = append(prepareArgs, "--experimental-patches", constants.PatchesDir)
}

if err := cp.Command(
Expand All @@ -192,7 +208,10 @@ func kubeadmJoinControlPlaneWithPhases(cp *status.Node, automaticCopyCerts bool,
fmt.Sprintf("--v=%d", vLevel),
}
if kustomizeDir != "" {
controlPlaneArgs = append(controlPlaneArgs, "-k", constants.KustomizeDir)
controlPlaneArgs = append(controlPlaneArgs, "-k", constants.PatchesDir)
}
if patchesDir != "" {
controlPlaneArgs = append(controlPlaneArgs, "--experimental-patches", constants.PatchesDir)
}
if automaticCopyCerts {
// if before v1.15, add certificate key flag (for >= 15, certificate key is passed via the config file)
Expand Down
30 changes: 23 additions & 7 deletions kinder/pkg/cluster/manager/actions/kubeadm-upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import (
//
// The implementation assumes that the kubeadm/kubelet/kubectl binaries and all the necessary images
// for the new kubernetes version are available in the /kinder/upgrade/{version} folder.
func KubeadmUpgrade(c *status.Cluster, upgradeVersion *K8sVersion.Version, kustomizeDir string, wait time.Duration, vLevel int) (err error) {
func KubeadmUpgrade(c *status.Cluster, upgradeVersion *K8sVersion.Version, kustomizeDir, patchesDir string, wait time.Duration, vLevel int) (err error) {
if upgradeVersion == nil {
return errors.New("kubeadm-upgrade actions requires the --upgrade-version parameter to be set")
}
Expand All @@ -57,14 +57,24 @@ func KubeadmUpgrade(c *status.Cluster, upgradeVersion *K8sVersion.Version, kusto
}
}

// if patcheDir is defined, copy the patches to the node
if patchesDir != "" {
if n.MustKubeadmVersion().LessThan(constants.V1_19) {
return errors.New("--patches can't be used with kubeadm older than v1.19")
}
if err := copyPatchesToNode(n, patchesDir); err != nil {
return err
}
}

if err := upgradeKubeadmBinary(n, upgradeVersion); err != nil {
return err
}

if n.Name() == c.BootstrapControlPlane().Name() {
err = kubeadmUpgradeApply(c, n, upgradeVersion, kustomizeDir, wait, vLevel)
err = kubeadmUpgradeApply(c, n, upgradeVersion, kustomizeDir, patchesDir, wait, vLevel)
} else {
err = kubeadmUpgradeNode(c, n, upgradeVersion, kustomizeDir, wait, vLevel)
err = kubeadmUpgradeNode(c, n, upgradeVersion, kustomizeDir, patchesDir, wait, vLevel)
}
if err != nil {
return err
Expand Down Expand Up @@ -129,12 +139,15 @@ func upgradeKubeadmBinary(n *status.Node, upgradeVersion *K8sVersion.Version) er
return nil
}

func kubeadmUpgradeApply(c *status.Cluster, cp1 *status.Node, upgradeVersion *K8sVersion.Version, kustomizeDir string, wait time.Duration, vLevel int) error {
func kubeadmUpgradeApply(c *status.Cluster, cp1 *status.Node, upgradeVersion *K8sVersion.Version, kustomizeDir, patchesDir string, wait time.Duration, vLevel int) error {
applyArgs := []string{
"upgrade", "apply", "-f", fmt.Sprintf("v%s", upgradeVersion), fmt.Sprintf("--v=%d", vLevel),
}
if kustomizeDir != "" {
applyArgs = append(applyArgs, fmt.Sprintf("-k=%s", constants.KustomizeDir))
applyArgs = append(applyArgs, fmt.Sprintf("-k=%s", constants.PatchesDir))
}
if patchesDir != "" {
applyArgs = append(applyArgs, "--experimental-patches", constants.PatchesDir)
}
if err := cp1.Command(
"kubeadm", applyArgs...,
Expand All @@ -149,7 +162,7 @@ func kubeadmUpgradeApply(c *status.Cluster, cp1 *status.Node, upgradeVersion *K8
return nil
}

func kubeadmUpgradeNode(c *status.Cluster, n *status.Node, upgradeVersion *K8sVersion.Version, kustomizeDir string, wait time.Duration, vLevel int) error {
func kubeadmUpgradeNode(c *status.Cluster, n *status.Node, upgradeVersion *K8sVersion.Version, kustomizeDir, patchesDir string, wait time.Duration, vLevel int) error {
// waitKubeletHasRBAC waits for the kubelet to have access to the expected config map
// please note that this is a temporary workaround for a problem we are observing on upgrades while
// executing node upgrades immediately after control-plane upgrade.
Expand All @@ -166,7 +179,10 @@ func kubeadmUpgradeNode(c *status.Cluster, n *status.Node, upgradeVersion *K8sVe
"upgrade", "node", fmt.Sprintf("--v=%d", vLevel),
}
if kustomizeDir != "" {
nodeArgs = append(nodeArgs, fmt.Sprintf("-k=%s", constants.KustomizeDir))
nodeArgs = append(nodeArgs, fmt.Sprintf("-k=%s", constants.PatchesDir))
}
if patchesDir != "" {
nodeArgs = append(nodeArgs, fmt.Sprintf("--experimental-patches=%s", constants.PatchesDir))
}
if err := n.Command(
"kubeadm", nodeArgs...,
Expand Down
7 changes: 5 additions & 2 deletions kinder/pkg/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,8 @@ const (
// DiscoveryFile defines the path to a discovery file stored on nodes
DiscoveryFile = "/kinder/discovery.conf"

// KustomizeDir defines the path to patches stored on node
KustomizeDir = "/kinder/kustomize"
// PatchesDir defines the path to patches stored on node
PatchesDir = "/kinder/patches"
)

// kubernetes releases, used for branching code according to K8s release or kubeadm release version
Expand All @@ -121,6 +121,9 @@ var (

// V1.18 minor version
V1_18 = K8sVersion.MustParseSemantic("v1.18.0-0")

// V1.19 minor version
V1_19 = K8sVersion.MustParseSemantic("v1.19.0-0")
)

// other constants
Expand Down

0 comments on commit 98e1626

Please sign in to comment.