Single Node deployment with bootstrap-in-place

cmd/cluster-bootstrap/bootstrapinplace.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"errors"
+
+	"github.com/openshift/cluster-bootstrap/pkg/bootstrapinplace"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	CmdBootstrapInPlace = &cobra.Command{
+		Use:          "bootstrap-in-place",
+		Short:        "Create Ignition based on Fedora CoreOS Config",
+		Long:         "",
+		PreRunE:      validateBootstrapInPlaceOpts,
+		RunE:         runCmdBootstrapInPlace,
+		SilenceUsage: true,
+	}
+
+	bootstrapInPlaceOpts struct {
+		assetDir     string
+		ignitionPath string
+		input        string
+		Strict       bool
+		Pretty       bool
+	}
+)
+
+func init() {
+	cmdRoot.AddCommand(CmdBootstrapInPlace)
+	CmdBootstrapInPlace.Flags().BoolVarP(&bootstrapInPlaceOpts.Strict, "strict", "s", true, "fail on any warning")
+	CmdBootstrapInPlace.Flags().BoolVarP(&bootstrapInPlaceOpts.Pretty, "pretty", "p", true, "output formatted json")
+	CmdBootstrapInPlace.Flags().StringVar(&bootstrapInPlaceOpts.input, "input", "", "fcc input file path")
+	CmdBootstrapInPlace.Flags().StringVar(&bootstrapInPlaceOpts.ignitionPath, "output", "o", "Ignition output file path")
+	CmdBootstrapInPlace.Flags().StringVarP(&bootstrapInPlaceOpts.assetDir, "asset-dir", "d", "", "allow embedding local files from this directory")
+
+}
+
+func runCmdBootstrapInPlace(cmd *cobra.Command, args []string) error {
+
+	bip, err := bootstrapinplace.NewBootstrapInPlaceCommand(bootstrapinplace.BootstrapInPlaceConfig{
+		AssetDir:     bootstrapInPlaceOpts.assetDir,
+		IgnitionPath: bootstrapInPlaceOpts.ignitionPath,
+		Input:        bootstrapInPlaceOpts.input,
+		Strict:       bootstrapInPlaceOpts.Strict,
+		Pretty:       bootstrapInPlaceOpts.Pretty,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return bip.Create()
+}
+
+func validateBootstrapInPlaceOpts(cmd *cobra.Command, args []string) error {
+	if bootstrapInPlaceOpts.ignitionPath == "" {
+		return errors.New("missing required flag: --output")
+	}
+	if bootstrapInPlaceOpts.assetDir == "" {
+		return errors.New("missing required flag: --asset-dir")
+	}
+	if bootstrapInPlaceOpts.input == "" {
+		return errors.New("missing required flag: --input")
+	}
+	return nil
+}

cmd/cluster-bootstrap/start.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -26,6 +27,7 @@ var (
 		requiredPodClauses   []string
 		waitForTearDownEvent string
 		earlyTearDown        bool
+		assetsCreatedTimeout time.Duration
 	}
 )
 
@@ -44,6 +46,7 @@ func init() {
 	cmdStart.Flags().StringSliceVar(&startOpts.requiredPodClauses, "required-pods", defaultRequiredPods, "List of pods name prefixes with their namespace (written as <namespace>/<pod-prefix>) that are required to be running and ready before the start command does the pivot, or alternatively a list of or'ed pod prefixes with a description (written as <desc>:<namespace>/<pod-prefix>|<namespace>/<pod-prefix>|...).")
 	cmdStart.Flags().StringVar(&startOpts.waitForTearDownEvent, "tear-down-event", "", "if this optional event name of the form <ns>/<event-name> is given, the event is waited for before tearing down the bootstrap control plane")
 	cmdStart.Flags().BoolVar(&startOpts.earlyTearDown, "tear-down-early", true, "tear down immediate after the non-bootstrap control plane is up and bootstrap-success event is created.")
+	cmdStart.Flags().DurationVar(&startOpts.assetsCreatedTimeout, "assets-create-timeout", time.Duration(60)*time.Minute, "how long to wait for all the assets be created.")
 }
 
 func runCmdStart(cmd *cobra.Command, args []string) error {
@@ -59,6 +62,7 @@ func runCmdStart(cmd *cobra.Command, args []string) error {
 		RequiredPodPrefixes:  podPrefixes,
 		WaitForTearDownEvent: startOpts.waitForTearDownEvent,
 		EarlyTearDown:        startOpts.earlyTearDown,
+		AssetsCreatedTimeout: startOpts.assetsCreatedTimeout,
 	})
 	if err != nil {
 		return err

pkg/bootstrapinplace/bootstrapinplace.go

@@ -0,0 +1,69 @@
+// Copyright 2019 Red Hat, Inc
+
+package bootstrapinplace
+
+import (
+	"io/ioutil"
+	"os"
+
+	"github.com/coreos/fcct/config"
+	fcctCommon "github.com/coreos/fcct/config/common"
+	"github.com/openshift/cluster-bootstrap/pkg/common"
+)
+
+func fail(format string, args ...interface{}) {
+	common.UserOutput(format, args...)
+	os.Exit(1)
+}
+
+type BootstrapInPlaceConfig struct {
+	AssetDir     string
+	IgnitionPath string
+	Input        string
+	Strict       bool
+	Pretty       bool
+}
+type BootstrapInPlaceCommand struct {
+	config BootstrapInPlaceConfig
+}
+
+func NewBootstrapInPlaceCommand(config BootstrapInPlaceConfig) (*BootstrapInPlaceCommand, error) {
+	return &BootstrapInPlaceCommand{
+		config: config,
+	}, nil
+}
+
+func (i *BootstrapInPlaceCommand) Create() error {
+
+	infile, err := os.Open(i.config.Input)
+	if err != nil {
+		fail("Error occurred while trying to open %s: %v\n", i.config.Input, err)
+	}
+	defer infile.Close()
+
+	dataIn, err := ioutil.ReadAll(infile)
+	if err != nil {
+		fail("Error occurred while trying to read %s: %v\n", infile.Name(), err)
+	}
+
+	dataOut, r, err := config.TranslateBytes(dataIn, fcctCommon.TranslateBytesOptions{
+		TranslateOptions: fcctCommon.TranslateOptions{FilesDir: i.config.AssetDir},
+		Pretty:           i.config.Pretty,
+		Strict:           i.config.Strict},
+	)
+	common.UserOutput("%s", r.String())
+	if err != nil {
+		fail("Error translating config: %v\n", err)
+	}
+
+	outfile, err := os.OpenFile(i.config.IgnitionPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		fail("Failed to open %s: %v\n", i.config.IgnitionPath, err)
+	}
+	defer outfile.Close()
+
+	if _, err := outfile.Write(append(dataOut, '\n')); err != nil {
+		fail("Failed to write config to %s: %v\n", outfile.Name(), err)
+	}
+	return nil
+}

pkg/common/common.go

@@ -0,0 +1,11 @@
+package common
+
+import "fmt"
+
+// All start command printing to stdout should go through this fmt.Printf wrapper.
+// The stdout of the start command should convey information useful to a human sitting
+// at a terminal watching their cluster bootstrap itself. Otherwise the message
+// should go to stderr.
+func UserOutput(format string, a ...interface{}) {
+	fmt.Printf(format, a...)
+}

pkg/start/bootstrap.go

@@ -1,30 +1,39 @@
 package start
 
 import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"github.com/openshift/cluster-bootstrap/pkg/common"
 	"io"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 type bootstrapControlPlane struct {
 	assetDir        string
 	podManifestPath string
 	ownedManifests  []string
+	kubeApiHost     string
 }
 
 // newBootstrapControlPlane constructs a new bootstrap control plane object.
-func newBootstrapControlPlane(assetDir, podManifestPath string) *bootstrapControlPlane {
+func newBootstrapControlPlane(assetDir, podManifestPath string, kubeApiHost string) *bootstrapControlPlane {
 	return &bootstrapControlPlane{
 		assetDir:        assetDir,
 		podManifestPath: podManifestPath,
+		kubeApiHost:     kubeApiHost,
 	}
 }
 
 // Start seeds static manifests to the kubelet to launch the bootstrap control plane.
 // Users should always ensure that Cleanup() is called even in the case of errors.
 func (b *bootstrapControlPlane) Start() error {
-	UserOutput("Starting temporary bootstrap control plane...\n")
+	common.UserOutput("Starting temporary bootstrap control plane...\n")
 	// Make secrets temporarily available to bootstrap cluster.
 	if err := os.RemoveAll(bootstrapSecretsDir); err != nil {
 		return err
@@ -42,7 +51,42 @@ func (b *bootstrapControlPlane) Start() error {
 	manifestsDir := filepath.Join(b.assetDir, assetPathBootstrapManifests)
 	ownedManifests, err := copyDirectory(manifestsDir, b.podManifestPath, false /* overwrite */)
 	b.ownedManifests = ownedManifests // always copy in case of partial failure.
-	return err
+	if err != nil {
+		return err
+	}
+
+	// Wait for kube-apiserver to be available and return.
+	return b.waitForApi()
+}
+
+// waitForApi will wait until kube-apiserver readyz endpoint is available
+func (b *bootstrapControlPlane) waitForApi() error {
+	common.UserOutput("Waiting up to %v for the Kubernetes API\n", bootstrapPodsRunningTimeout)
+	apiContext, cancel := context.WithTimeout(context.Background(), bootstrapPodsRunningTimeout)
+	defer cancel()
+	customTransport := http.DefaultTransport.(*http.Transport).Clone()
+	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	client := &http.Client{Transport: customTransport}
+	previousErrorSuffix := ""
+	wait.Until(func() {
+		_, err := client.Get(fmt.Sprintf("https://%s/readyz", b.kubeApiHost))
+		if err == nil {
+			common.UserOutput("API is up\n")
+			cancel()
+		} else {
+			chunks := strings.Split(err.Error(), ":")
+			errorSuffix := chunks[len(chunks)-1]
+			if previousErrorSuffix != errorSuffix {
+				common.UserOutput("Still waiting for the Kubernetes API: %v\n", err)
+				previousErrorSuffix = errorSuffix
+			}
+		}
+	}, time.Second, apiContext.Done())
+	if apiContext.Err() == context.Canceled {
+		return nil
+	} else {
+		return fmt.Errorf("time out waiting for Kubernetes API")
+	}
 }
 
 // Teardown brings down the bootstrap control plane and cleans up the temporary manifests and
@@ -52,7 +96,7 @@ func (b *bootstrapControlPlane) Teardown() error {
 		return nil
 	}
 
-	UserOutput("Tearing down temporary bootstrap control plane...\n")
+	common.UserOutput("Tearing down temporary bootstrap control plane...\n")
 	if err := os.RemoveAll(bootstrapSecretsDir); err != nil {
 		return err
 	}

pkg/start/bootstrap_test.go

@@ -1,10 +1,14 @@
 package start
 
 import (
+	"fmt"
 	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"reflect"
+	"strings"
 	"testing"
 )
 
@@ -13,6 +17,13 @@ var (
 	manifests = []string{"pod-1.yaml", "pod-2.yaml"}
 )
 
+func createTestServer() (*httptest.Server, string) {
+	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "ok")
+	}))
+	return ts, strings.Replace(ts.URL, "https://", "", 1)
+}
+
 func setUp(t *testing.T) (assetDir, podManifestPath string) {
 	// Create source directories.
 	var err error
@@ -71,8 +82,11 @@ func TestBootstrapControlPlane(t *testing.T) {
 	assetDir, podManifestPath := setUp(t)
 	defer tearDown(assetDir, podManifestPath, t)
 
+	ts, url := createTestServer()
+	defer ts.Close()
+
 	// Create and start bootstrap control plane.
-	bcp := newBootstrapControlPlane(assetDir, podManifestPath)
+	bcp := newBootstrapControlPlane(assetDir, podManifestPath, url)
 	if err := bcp.Start(); err != nil {
 		t.Errorf("bcp.Start() = %v, want: nil", err)
 	}
@@ -117,7 +131,7 @@ func TestBootstrapControlPlaneNoOverwrite(t *testing.T) {
 	}
 
 	// Create and start bootstrap control plane.
-	bcp := newBootstrapControlPlane(assetDir, podManifestPath)
+	bcp := newBootstrapControlPlane(assetDir, podManifestPath, "")
 	if err := bcp.Start(); err == nil {
 		t.Errorf("bcp.Start() = %v, want: non-nil", err)
 	}