Single Node deployment with bootstrap-in-place

cmd/cluster-bootstrap/bootstrapinplace.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"errors"
+
+	"github.com/openshift/cluster-bootstrap/pkg/bootstrapinplace"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	CmdBootstrapInPlace = &cobra.Command{
+		Use:          "bootstrap-in-place",
+		Short:        "Create Ignition based on Fedora CoreOS Config",
+		Long:         "",
+		PreRunE:      validateBootstrapInPlaceOpts,
+		RunE:         runCmdBootstrapInPlace,
+		SilenceUsage: true,
+	}
+
+	bootstrapInPlaceOpts struct {
+		assetDir     string
+		ignitionPath string
+		input        string
+		Strict       bool
+		Pretty       bool
+	}
+)
+
+func init() {
+	cmdRoot.AddCommand(CmdBootstrapInPlace)
+	CmdBootstrapInPlace.Flags().BoolVarP(&bootstrapInPlaceOpts.Strict, "strict", "s", true, "fail on any warning")
+	CmdBootstrapInPlace.Flags().BoolVarP(&bootstrapInPlaceOpts.Pretty, "pretty", "p", true, "output formatted json")
+	CmdBootstrapInPlace.Flags().StringVar(&bootstrapInPlaceOpts.input, "input", "", "fcc input file path")
+	CmdBootstrapInPlace.Flags().StringVar(&bootstrapInPlaceOpts.ignitionPath, "output", "o", "Ignition output file path")
+	CmdBootstrapInPlace.Flags().StringVarP(&bootstrapInPlaceOpts.assetDir, "asset-dir", "d", "", "allow embedding local files from this directory")
+}
+
+func runCmdBootstrapInPlace(cmd *cobra.Command, args []string) error {
+	bip, err := bootstrapinplace.NewBootstrapInPlaceCommand(bootstrapinplace.BootstrapInPlaceConfig{
+		AssetDir:     bootstrapInPlaceOpts.assetDir,
+		IgnitionPath: bootstrapInPlaceOpts.ignitionPath,
+		Input:        bootstrapInPlaceOpts.input,
+		Strict:       bootstrapInPlaceOpts.Strict,
+		Pretty:       bootstrapInPlaceOpts.Pretty,
+	})
+	if err != nil {
+		return err
+	}
+
+	return bip.Create()
+}
+
+func validateBootstrapInPlaceOpts(cmd *cobra.Command, args []string) error {
+	if bootstrapInPlaceOpts.ignitionPath == "" {
+		return errors.New("missing required flag: --output")
+	}
+	if bootstrapInPlaceOpts.assetDir == "" {
+		return errors.New("missing required flag: --asset-dir")
+	}
+	if bootstrapInPlaceOpts.input == "" {
+		return errors.New("missing required flag: --input")
+	}
+	return nil
+}

cmd/cluster-bootstrap/start.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -26,6 +27,7 @@ var (
 		requiredPodClauses   []string
 		waitForTearDownEvent string
 		earlyTearDown        bool
+		assetsCreatedTimeout time.Duration
 	}
 )
 
@@ -44,6 +46,7 @@ func init() {
 	cmdStart.Flags().StringSliceVar(&startOpts.requiredPodClauses, "required-pods", defaultRequiredPods, "List of pods name prefixes with their namespace (written as <namespace>/<pod-prefix>) that are required to be running and ready before the start command does the pivot, or alternatively a list of or'ed pod prefixes with a description (written as <desc>:<namespace>/<pod-prefix>|<namespace>/<pod-prefix>|...).")
 	cmdStart.Flags().StringVar(&startOpts.waitForTearDownEvent, "tear-down-event", "", "if this optional event name of the form <ns>/<event-name> is given, the event is waited for before tearing down the bootstrap control plane")
 	cmdStart.Flags().BoolVar(&startOpts.earlyTearDown, "tear-down-early", true, "tear down immediate after the non-bootstrap control plane is up and bootstrap-success event is created.")
+	cmdStart.Flags().DurationVar(&startOpts.assetsCreatedTimeout, "assets-create-timeout", time.Duration(60)*time.Minute, "how long to wait for all the assets be created.")
 }
 
 func runCmdStart(cmd *cobra.Command, args []string) error {
@@ -59,6 +62,7 @@ func runCmdStart(cmd *cobra.Command, args []string) error {
 		RequiredPodPrefixes:  podPrefixes,
 		WaitForTearDownEvent: startOpts.waitForTearDownEvent,
 		EarlyTearDown:        startOpts.earlyTearDown,
+		AssetsCreatedTimeout: startOpts.assetsCreatedTimeout,
 	})
 	if err != nil {
 		return err

go.mod

@@ -3,8 +3,8 @@ module github.com/openshift/cluster-bootstrap
 go 1.13
 
 require (
+	github.com/coreos/fcct v0.9.0
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/googleapis/gnostic v0.5.1 // indirect
 	github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab
 	github.com/openshift/library-go v0.0.0-20200930190915-f7cb85f605db

pkg/bootstrapinplace/bootstrapinplace.go

@@ -0,0 +1,73 @@
+// Copyright 2019 Red Hat, Inc
+
+package bootstrapinplace
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/coreos/fcct/config"
+	fcctCommon "github.com/coreos/fcct/config/common"
+)
+
+func fail(format string, args ...interface{}) {
+	fmt.Printf(format, args...)
+	os.Exit(1)
+}
+
+type BootstrapInPlaceConfig struct {
+	AssetDir     string
+	IgnitionPath string
+	Input        string
+	Strict       bool
+	Pretty       bool
+}
+type BootstrapInPlaceCommand struct {
+	config BootstrapInPlaceConfig
+}
+
+func NewBootstrapInPlaceCommand(config BootstrapInPlaceConfig) (*BootstrapInPlaceCommand, error) {
+	return &BootstrapInPlaceCommand{
+		config: config,
+	}, nil
+}
+
+// Creating master ignition that will be used by node after reboot
+// Using fcct tool (tool that takes yaml and according to it creates ignition):
+// 1. Read actions yaml that has all the data needed by fcct to create master.ign
+// 2. Create ignition data
+// 3. Write created data to file
+func (i *BootstrapInPlaceCommand) Create() error {
+	infile, err := os.Open(i.config.Input)
+	if err != nil {
+		fail("Error occurred while trying to open %s: %v\n", i.config.Input, err)
+	}
+	defer infile.Close()
+
+	dataIn, err := ioutil.ReadAll(infile)
+	if err != nil {
+		fail("Error occurred while trying to read %s: %v\n", infile.Name(), err)
+	}
+
+	dataOut, r, err := config.TranslateBytes(dataIn, fcctCommon.TranslateBytesOptions{
+		TranslateOptions: fcctCommon.TranslateOptions{FilesDir: i.config.AssetDir},
+		Pretty:           i.config.Pretty,
+		Strict:           i.config.Strict},
+	)
+	fmt.Println(r.String())
+	if err != nil {
+		fail("Error translating config: %v\n", err)
+	}
+
+	outfile, err := os.OpenFile(i.config.IgnitionPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		fail("Failed to open %s: %v\n", i.config.IgnitionPath, err)
+	}
+	defer outfile.Close()
+
+	if _, err := outfile.Write(append(dataOut, '\n')); err != nil {
+		fail("Failed to write config to %s: %v\n", outfile.Name(), err)
+	}
+	return nil
+}

pkg/start/bootstrap.go

@@ -1,23 +1,31 @@
 package start
 
 import (
+	"context"
+	"crypto/tls"
+	"fmt"
 	"io"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 type bootstrapControlPlane struct {
 	assetDir        string
 	podManifestPath string
 	ownedManifests  []string
+	kubeApiHost     string
 }
 
 // newBootstrapControlPlane constructs a new bootstrap control plane object.
-func newBootstrapControlPlane(assetDir, podManifestPath string) *bootstrapControlPlane {
+func newBootstrapControlPlane(assetDir, podManifestPath string, kubeApiHost string) *bootstrapControlPlane {
 	return &bootstrapControlPlane{
 		assetDir:        assetDir,
 		podManifestPath: podManifestPath,
+		kubeApiHost:     kubeApiHost,
 	}
 }
 
@@ -42,7 +50,39 @@ func (b *bootstrapControlPlane) Start() error {
 	manifestsDir := filepath.Join(b.assetDir, assetPathBootstrapManifests)
 	ownedManifests, err := copyDirectory(manifestsDir, b.podManifestPath, false /* overwrite */)
 	b.ownedManifests = ownedManifests // always copy in case of partial failure.
-	return err
+	if err != nil {
+		return err
+	}
+
+	// Wait for kube-apiserver to be available and return.
+	return b.waitForApi()
+}
+
+// waitForApi will wait until kube-apiserver readyz endpoint is available
+func (b *bootstrapControlPlane) waitForApi() error {
+	UserOutput("Waiting up to %v for the Kubernetes API\n", bootstrapPodsRunningTimeout)
+	apiContext, cancel := context.WithTimeout(context.Background(), bootstrapPodsRunningTimeout)
+	defer cancel()
+	customTransport := http.DefaultTransport.(*http.Transport).Clone()
+	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	client := &http.Client{Transport: customTransport}
+	previousError := ""
+	err := wait.PollUntil(time.Second, func() (bool, error) {
+		if _, err := client.Get(fmt.Sprintf("https://%s/readyz", b.kubeApiHost)); err == nil {
+			UserOutput("API is up\n")
+			return true, nil
+		} else if previousError != err.Error() {
+			UserOutput("Still waiting for the Kubernetes API: %v \n", err)
+			previousError = err.Error()
+		}
+
+		return false, nil
+	}, apiContext.Done())
+	if err != nil {
+		return fmt.Errorf("time out waiting for Kubernetes API")
+	}
+
+	return nil
 }
 
 // Teardown brings down the bootstrap control plane and cleans up the temporary manifests and

pkg/start/bootstrap_test.go

@@ -1,10 +1,15 @@
 package start
 
 import (
+	"fmt"
 	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"reflect"
+	"strings"
 	"testing"
 )
 
@@ -13,6 +18,17 @@ var (
 	manifests = []string{"pod-1.yaml", "pod-2.yaml"}
 )
 
+func createTestServer() (*httptest.Server, string) {
+	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if rand.Intn(2) == 1 {
+			panic("Randomly created error")
+		}
+		fmt.Fprintln(w, "ok")
+	}))
+	return ts, strings.Replace(ts.URL, "https://", "", 1)
+}
+
 func setUp(t *testing.T) (assetDir, podManifestPath string) {
 	// Create source directories.
 	var err error
@@ -71,8 +87,11 @@ func TestBootstrapControlPlane(t *testing.T) {
 	assetDir, podManifestPath := setUp(t)
 	defer tearDown(assetDir, podManifestPath, t)
 
+	ts, url := createTestServer()
+	defer ts.Close()
+
 	// Create and start bootstrap control plane.
-	bcp := newBootstrapControlPlane(assetDir, podManifestPath)
+	bcp := newBootstrapControlPlane(assetDir, podManifestPath, url)
 	if err := bcp.Start(); err != nil {
 		t.Errorf("bcp.Start() = %v, want: nil", err)
 	}
@@ -117,7 +136,7 @@ func TestBootstrapControlPlaneNoOverwrite(t *testing.T) {
 	}
 
 	// Create and start bootstrap control plane.
-	bcp := newBootstrapControlPlane(assetDir, podManifestPath)
+	bcp := newBootstrapControlPlane(assetDir, podManifestPath, "")
 	if err := bcp.Start(); err == nil {
 		t.Errorf("bcp.Start() = %v, want: non-nil", err)
 	}

pkg/start/start.go

@@ -24,8 +24,6 @@ import (
 const (
 	// how long we wait until the bootstrap pods to be running
 	bootstrapPodsRunningTimeout = 20 * time.Minute
-	// how long we wait until the assets must all be created
-	assetsCreatedTimeout = 60 * time.Minute
 )
 
 type Config struct {
@@ -35,6 +33,7 @@ type Config struct {
 	RequiredPodPrefixes  map[string][]string
 	WaitForTearDownEvent string
 	EarlyTearDown        bool
+	AssetsCreatedTimeout time.Duration
 }
 
 type startCommand struct {
@@ -44,6 +43,7 @@ type startCommand struct {
 	requiredPodPrefixes  map[string][]string
 	waitForTearDownEvent string
 	earlyTearDown        bool
+	assetsCreatedTimeout time.Duration
 }
 
 func NewStartCommand(config Config) (*startCommand, error) {
@@ -54,6 +54,7 @@ func NewStartCommand(config Config) (*startCommand, error) {
 		requiredPodPrefixes:  config.RequiredPodPrefixes,
 		waitForTearDownEvent: config.WaitForTearDownEvent,
 		earlyTearDown:        config.EarlyTearDown,
+		assetsCreatedTimeout: config.AssetsCreatedTimeout,
 	}, nil
 }
 
@@ -67,7 +68,12 @@ func (b *startCommand) Run() error {
 		return err
 	}
 
-	bcp := newBootstrapControlPlane(b.assetDir, b.podManifestPath)
+	// We don't want the client contact the API servers via load-balancer, but only talk to the local API server.
+	// This will speed up the initial "where is working API server" process.
+	localClientConfig := rest.CopyConfig(restConfig)
+	localClientConfig.Host = "localhost:6443"
+
+	bcp := newBootstrapControlPlane(b.assetDir, b.podManifestPath, localClientConfig.Host)
 
 	// Always tear down the bootstrap control plane and clean up manifests and secrets.
 	defer func() {
@@ -87,10 +93,6 @@ func (b *startCommand) Run() error {
 		return err
 	}
 
-	// We don't want the client contact the API servers via load-balancer, but only talk to the local API server.
-	// This will speed up the initial "where is working API server" process.
-	localClientConfig := rest.CopyConfig(restConfig)
-	localClientConfig.Host = "localhost:6443"
 	// Set the ServerName to original hostname so we pass the certificate check.
 	hostURL, err := url.Parse(restConfig.Host)
 	if err != nil {
@@ -137,7 +139,7 @@ func (b *startCommand) Run() error {
 	}
 
 	// continue with assets
-	ctx, cancel = context.WithTimeout(context.Background(), assetsCreatedTimeout)
+	ctx, cancel = context.WithTimeout(context.Background(), b.assetsCreatedTimeout)
 	defer cancel()
 	if b.earlyTearDown {
 		// switch over to ELB client and continue with the assets
@@ -174,6 +176,10 @@ func (b *startCommand) Run() error {
 	UserOutput("Waiting for remaining assets to be created.\n")
 	assetsDone.Wait()
 
+	// We want to fail in case we failed to create some manifests
+	if ctx.Err() == context.DeadlineExceeded {
+		return fmt.Errorf("timed out creating manifests")
+	}
 	UserOutput("Sending bootstrap-finished event.")
 	if _, err := client.CoreV1().Events("kube-system").Create(context.Background(), makeBootstrapSuccessEvent("kube-system", "bootstrap-finished"), metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
 		return err