diff --git a/cmd/cluster-bootstrap/ibip.go b/cmd/cluster-bootstrap/ibip.go new file mode 100644 index 000000000..212badcf0 --- /dev/null +++ b/cmd/cluster-bootstrap/ibip.go @@ -0,0 +1,54 @@ +package main + +import ( + "errors" + "github.com/openshift/cluster-bootstrap/pkg/ibip" + "github.com/spf13/cobra" + +) + +var ( + cmdIBip = &cobra.Command{ + Use: "ibip", + Short: "Update the master ignition with control plane static pods files", + Long: "", + PreRunE: validateIBipOpts, + RunE: runCmdIBip, + SilenceUsage: true, + } + + iBipOpts struct { + assetDir string + ignitionPath string + } +) + +func init() { + cmdRoot.AddCommand(cmdIBip) + cmdIBip.Flags().StringVar(&iBipOpts.assetDir, "asset-dir", "", "Path to the cluster asset directory.") + cmdIBip.Flags().StringVar(&iBipOpts.ignitionPath, "ignition-path", "/assets/master.ign", "The location of master ignition") + +} + +func runCmdIBip(cmd *cobra.Command, args []string) error { + + ib, err := ibip.NewIBipCommand(ibip.ConfigIBip{ + AssetDir: iBipOpts.assetDir, + IgnitionPath: iBipOpts.ignitionPath, + }) + if err != nil { + return err + } + + return ib.UpdateSnoIgnitionData() +} + +func validateIBipOpts(cmd *cobra.Command, args []string) error { + if iBipOpts.ignitionPath == "" { + return errors.New("missing required flag: --ignition-path") + } + if iBipOpts.assetDir == "" { + return errors.New("missing required flag: --asset-dir") + } + return nil +} diff --git a/cmd/cluster-bootstrap/start.go b/cmd/cluster-bootstrap/start.go index 3934f7e65..e5bf53fa2 100644 --- a/cmd/cluster-bootstrap/start.go +++ b/cmd/cluster-bootstrap/start.go @@ -26,6 +26,7 @@ var ( requiredPodClauses []string waitForTearDownEvent string earlyTearDown bool + clusterProfile string } ) @@ -44,6 +45,7 @@ func init() { cmdStart.Flags().StringSliceVar(&startOpts.requiredPodClauses, "required-pods", defaultRequiredPods, "List of pods name prefixes with their namespace (written as /) that are required to be running and ready before the start command does the pivot, or alternatively a list of or'ed pod prefixes with a description (written as :/|/|...).") cmdStart.Flags().StringVar(&startOpts.waitForTearDownEvent, "tear-down-event", "", "if this optional event name of the form / is given, the event is waited for before tearing down the bootstrap control plane") cmdStart.Flags().BoolVar(&startOpts.earlyTearDown, "tear-down-early", true, "tear down immediate after the non-bootstrap control plane is up and bootstrap-success event is created.") + cmdStart.Flags().StringVar(&startOpts.clusterProfile, "cluster-profile", "", "The cluster profile.") } func runCmdStart(cmd *cobra.Command, args []string) error { @@ -59,6 +61,7 @@ func runCmdStart(cmd *cobra.Command, args []string) error { RequiredPodPrefixes: podPrefixes, WaitForTearDownEvent: startOpts.waitForTearDownEvent, EarlyTearDown: startOpts.earlyTearDown, + ClusterProfile: startOpts.clusterProfile, }) if err != nil { return err diff --git a/pkg/common/common.go b/pkg/common/common.go new file mode 100644 index 000000000..7f2cd2593 --- /dev/null +++ b/pkg/common/common.go @@ -0,0 +1,11 @@ +package common + +import "fmt" + +// All start command printing to stdout should go through this fmt.Printf wrapper. +// The stdout of the start command should convey information useful to a human sitting +// at a terminal watching their cluster bootstrap itself. Otherwise the message +// should go to stderr. +func UserOutput(format string, a ...interface{}) { + fmt.Printf(format, a...) +} diff --git a/pkg/ibip/ibip.go b/pkg/ibip/ibip.go new file mode 100644 index 000000000..d356e900f --- /dev/null +++ b/pkg/ibip/ibip.go @@ -0,0 +1,215 @@ +package ibip + +import ( + "encoding/base64" + "encoding/json" + "github.com/openshift/cluster-bootstrap/pkg/common" + "io/ioutil" + "os" + "path/filepath" + "strings" + + ignition "github.com/coreos/ignition/v2/config/v3_1" + ignitionTypes "github.com/coreos/ignition/v2/config/v3_1/types" +) + +type ConfigIBip struct { + AssetDir string + IgnitionPath string +} + +type iBipCommand struct { + ignitionPath string + assetDir string +} + + +func NewIBipCommand(config ConfigIBip) (*iBipCommand, error) { + return &iBipCommand{ + assetDir: config.AssetDir, + ignitionPath: config.IgnitionPath, + }, nil +} + +const ( + kubeDir = "/etc/kubernetes" + assetPathBootstrapManifests = "bootstrap-manifests" + manifests = "manifests" + bootstrapConfigs = "bootstrap-configs" + bootstrapSecrets = "bootstrap-secrets" + etcdDataDir = "/var/lib/etcd" +) + +type ignitionFile struct { + filePath string + fileContents string + mode int +} + +type filesToGather struct { + pathForSearch string + pattern string + ignitionPath string +} + +func newIgnitionFile(path string, filePathInIgnition string) (*ignitionFile, error) { + content, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + encodedContent := "data:text/plain;charset=utf-8;base64," + base64.StdEncoding.EncodeToString(content) + return &ignitionFile{filePath: filePathInIgnition, + mode: 420, fileContents: encodedContent}, nil +} + +func (i *iBipCommand) createListOfIgnitionFiles(files []string, searchedFolder string, folderInIgnition string) ([]*ignitionFile, error) { + var ignitionFiles []*ignitionFile + for _, path := range files { + // Take relative path + filePath := filepath.Join(folderInIgnition, strings.ReplaceAll(path, searchedFolder, "")) + fileToAdd, err := newIgnitionFile(path, filePath) + if err != nil { + common.UserOutput("Failed to read %s", path) + return nil, err + } + ignitionFiles = append(ignitionFiles, fileToAdd) + } + return ignitionFiles, nil +} + +func (i *iBipCommand) createFilesList(filesToGatherList []filesToGather) ([]*ignitionFile, error) { + var fullList []*ignitionFile + for _, ft := range filesToGatherList { + files, err := i.findFiles(ft.pathForSearch, ft.pattern) + if err != nil { + common.UserOutput("Failed to search for files in %s with pattern %s, err %e", ft.pathForSearch, ft.pattern, err) + return nil, err + } + ignitionFiles, err := i.createListOfIgnitionFiles(files, ft.pathForSearch, ft.ignitionPath) + if err != nil { + common.UserOutput("Failed to create ignitionsFile list for in %s with ign path %s, err %e", ft.pathForSearch, ft.ignitionPath, err) + return nil, err + } + fullList = append(fullList, ignitionFiles...) + } + return fullList, nil +} + +func (i *iBipCommand) UpdateSnoIgnitionData() error { + + common.UserOutput("Creating ignition file objects from required folders") + filesFromFolders := []filesToGather{ + {pathForSearch: filepath.Join(i.assetDir, assetPathBootstrapManifests), pattern: "kube*", ignitionPath: filepath.Join(kubeDir, manifests)}, + {pathForSearch: filepath.Join(kubeDir, bootstrapConfigs), pattern: "*", ignitionPath: filepath.Join(kubeDir, bootstrapConfigs)}, + {pathForSearch: filepath.Join(i.assetDir, "tls"), pattern: "*", ignitionPath: filepath.Join(kubeDir, bootstrapSecrets)}, + {pathForSearch: filepath.Join(i.assetDir, "etcd-bootstrap/bootstrap-manifests/secrets"), pattern: "*", ignitionPath: filepath.Join(kubeDir, "static-pod-resources/etcd-member")}, + {pathForSearch: etcdDataDir, pattern: "*", ignitionPath: etcdDataDir}, + } + + ignitionFileObjects, err := i.createFilesList(filesFromFolders) + if err != nil { + return err + } + + common.UserOutput("Creating ignition file objects from files that require rename") + singleFilesWithNameChange := map[string]string{ + filepath.Join(i.assetDir, "auth/kubeconfig-loopback"): filepath.Join(kubeDir, bootstrapSecrets+"/kubeconfig"), + filepath.Join(i.assetDir, "tls/etcd-ca-bundle.crt"): filepath.Join(kubeDir, "static-pod-resources/etcd-member/ca.crt"), + filepath.Join(i.assetDir, "etcd-bootstrap/bootstrap-manifests/etcd-member-pod.yaml"): filepath.Join(kubeDir, manifests+"/etcd-pod.yaml"), + } + + for path, ignPath := range singleFilesWithNameChange { + fileToAdd, err := newIgnitionFile(path, ignPath) + if err != nil { + common.UserOutput("Error occurred while trying to create ignitionFile from %s with ign path %s, err : %e", path, ignPath, err) + return err + } + ignitionFileObjects = append(ignitionFileObjects, fileToAdd) + } + + common.UserOutput("Ignition Path %s", i.ignitionPath) + err = i.addFilesToIgnitionFile(i.ignitionPath, ignitionFileObjects) + if err != nil { + common.UserOutput("Error occurred while trying to read %s : %e", i.ignitionPath, err) + return err + } + + return nil +} + +func (i *iBipCommand) addFilesToIgnitionObject(ignitionData []byte, files []*ignitionFile) ([]byte, error) { + + ignitionOutput, _, err := ignition.Parse(ignitionData) + if err != nil { + return nil, err + } + + for _, file := range files { + common.UserOutput("Adding file %s", file.filePath) + rootUser := "root" + iFile := ignitionTypes.File{ + Node: ignitionTypes.Node{ + Path: file.filePath, + Overwrite: nil, + Group: ignitionTypes.NodeGroup{}, + User: ignitionTypes.NodeUser{Name: &rootUser}, + }, + FileEmbedded1: ignitionTypes.FileEmbedded1{ + Append: []ignitionTypes.Resource{}, + Contents: ignitionTypes.Resource{ + Source: &file.fileContents, + }, + Mode: &file.mode, + }, + } + ignitionOutput.Storage.Files = append(ignitionOutput.Storage.Files, iFile) + } + return json.Marshal(ignitionOutput) +} + +func (i *iBipCommand) addFilesToIgnitionFile(ignitionPath string, files []*ignitionFile) error { + common.UserOutput("Adding files %d to ignition %s", len(files), ignitionPath) + ignitionData, err := ioutil.ReadFile(ignitionPath) + if err != nil { + common.UserOutput("Error occurred while trying to read %s : %e", ignitionPath, err) + return err + } + newIgnitionData, err := i.addFilesToIgnitionObject(ignitionData, files) + if err != nil { + common.UserOutput("Failed to write new ignition to %s : %e", ignitionPath, err) + return err + } + + err = ioutil.WriteFile(ignitionPath, newIgnitionData, os.ModePerm) + if err != nil { + common.UserOutput("Failed to write new ignition to %s", ignitionPath) + return err + } + + return nil +} + +func (i *iBipCommand) findFiles(root string, pattern string) ([]string, error) { + var matches []string + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path == root { + return nil + } + if info.IsDir() { + return nil + } + if matched, err := filepath.Match(pattern, filepath.Base(path)); err != nil { + return err + } else if matched { + matches = append(matches, path) + } + return nil + }) + if err != nil { + return nil, err + } + return matches, nil +} \ No newline at end of file diff --git a/pkg/start/bootstrap.go b/pkg/start/bootstrap.go index 9d4f71aee..4832d8de0 100644 --- a/pkg/start/bootstrap.go +++ b/pkg/start/bootstrap.go @@ -5,6 +5,8 @@ import ( "os" "path/filepath" "strings" + + "github.com/openshift/cluster-bootstrap/pkg/common" ) type bootstrapControlPlane struct { @@ -24,7 +26,7 @@ func newBootstrapControlPlane(assetDir, podManifestPath string) *bootstrapContro // Start seeds static manifests to the kubelet to launch the bootstrap control plane. // Users should always ensure that Cleanup() is called even in the case of errors. func (b *bootstrapControlPlane) Start() error { - UserOutput("Starting temporary bootstrap control plane...\n") + common.UserOutput("Starting temporary bootstrap control plane...\n") // Make secrets temporarily available to bootstrap cluster. if err := os.RemoveAll(bootstrapSecretsDir); err != nil { return err @@ -52,7 +54,7 @@ func (b *bootstrapControlPlane) Teardown() error { return nil } - UserOutput("Tearing down temporary bootstrap control plane...\n") + common.UserOutput("Tearing down temporary bootstrap control plane...\n") if err := os.RemoveAll(bootstrapSecretsDir); err != nil { return err } diff --git a/pkg/start/start.go b/pkg/start/start.go index bcf27b746..a4ef27c75 100644 --- a/pkg/start/start.go +++ b/pkg/start/start.go @@ -2,7 +2,9 @@ package start import ( "context" + "errors" "fmt" + "github.com/openshift/cluster-bootstrap/pkg/common" "net" "net/url" "os" @@ -25,7 +27,8 @@ const ( // how long we wait until the bootstrap pods to be running bootstrapPodsRunningTimeout = 20 * time.Minute // how long we wait until the assets must all be created - assetsCreatedTimeout = 60 * time.Minute + assetsCreatedTimeout = 60 * time.Minute + SingleNodeProductionEdge = "single-node-production-edge" ) type Config struct { @@ -35,6 +38,7 @@ type Config struct { RequiredPodPrefixes map[string][]string WaitForTearDownEvent string EarlyTearDown bool + ClusterProfile string } type startCommand struct { @@ -44,6 +48,7 @@ type startCommand struct { requiredPodPrefixes map[string][]string waitForTearDownEvent string earlyTearDown bool + clusterProfile string } func NewStartCommand(config Config) (*startCommand, error) { @@ -54,6 +59,7 @@ func NewStartCommand(config Config) (*startCommand, error) { requiredPodPrefixes: config.RequiredPodPrefixes, waitForTearDownEvent: config.WaitForTearDownEvent, earlyTearDown: config.EarlyTearDown, + clusterProfile: config.ClusterProfile, }, nil } @@ -72,14 +78,14 @@ func (b *startCommand) Run() error { // Always tear down the bootstrap control plane and clean up manifests and secrets. defer func() { if err := bcp.Teardown(); err != nil { - UserOutput("Error tearing down temporary bootstrap control plane: %v\n", err) + common.UserOutput("Error tearing down temporary bootstrap control plane: %v\n", err) } }() defer func() { // Always report errors. if err != nil { - UserOutput("Error: %v\n", err) + common.UserOutput("Error: %v\n", err) } }() @@ -114,7 +120,7 @@ func (b *startCommand) Run() error { select { case <-ctx.Done(): default: - UserOutput("Assert creation failed: %v\n", err) + common.UserOutput("Assert creation failed: %v\n", err) cancel() } } @@ -127,11 +133,20 @@ func (b *startCommand) Run() error { if err = waitUntilPodsRunning(ctx, client, b.requiredPodPrefixes); err != nil { return err } - cancel() - assetsDone.Wait() + if b.clusterProfile == SingleNodeProductionEdge { + // We don't want to cancel bcp since we are need it for applying the manifests + assetsDone.Wait() + // We want to fail fast in case we failed to apply some manifests + if ctx.Err() == context.DeadlineExceeded { + return errors.New("Timed out applying manifests") + } + } else { + cancel() + assetsDone.Wait() + } // notify installer that we are ready to tear down the temporary bootstrap control plane - UserOutput("Sending bootstrap-success event.") + common.UserOutput("Sending bootstrap-success event.") if _, err := client.CoreV1().Events("kube-system").Create(makeBootstrapSuccessEvent("kube-system", "bootstrap-success")); err != nil && !apierrors.IsAlreadyExists(err) { return err } @@ -158,7 +173,7 @@ func (b *startCommand) Run() error { if err := waitForEvent(context.TODO(), client, ns, name); err != nil { return err } - UserOutput("Got %s event.", b.waitForTearDownEvent) + common.UserOutput("Got %s event.", b.waitForTearDownEvent) } // tear down the bootstrap control plane. Set bcp to nil to avoid a second tear down in the defer func. @@ -166,15 +181,15 @@ func (b *startCommand) Run() error { err = bcp.Teardown() bcp = nil if err != nil { - UserOutput("Error tearing down temporary bootstrap control plane: %v\n", err) + common.UserOutput("Error tearing down temporary bootstrap control plane: %v\n", err) } } // wait for the tail of assets to be created after tear down - UserOutput("Waiting for remaining assets to be created.\n") + common.UserOutput("Waiting for remaining assets to be created.\n") assetsDone.Wait() - UserOutput("Sending bootstrap-finished event.") + common.UserOutput("Sending bootstrap-finished event.") if _, err := client.CoreV1().Events("kube-system").Create(makeBootstrapSuccessEvent("kube-system", "bootstrap-finished")); err != nil && !apierrors.IsAlreadyExists(err) { return err } @@ -182,20 +197,12 @@ func (b *startCommand) Run() error { return nil } -// All start command printing to stdout should go through this fmt.Printf wrapper. -// The stdout of the start command should convey information useful to a human sitting -// at a terminal watching their cluster bootstrap itself. Otherwise the message -// should go to stderr. -func UserOutput(format string, a ...interface{}) { - fmt.Printf(format, a...) -} - func waitForEvent(ctx context.Context, client kubernetes.Interface, ns, name string) error { return wait.PollImmediateUntil(time.Second, func() (done bool, err error) { if _, err := client.CoreV1().Events(ns).Get(name, metav1.GetOptions{}); err != nil && apierrors.IsNotFound(err) { return false, nil } else if err != nil { - UserOutput("Error waiting for %s/%s event: %v", ns, name, err) + common.UserOutput("Error waiting for %s/%s event: %v", ns, name, err) return false, nil } return true, nil diff --git a/pkg/start/status.go b/pkg/start/status.go index 84d150fec..174d49d98 100644 --- a/pkg/start/status.go +++ b/pkg/start/status.go @@ -3,6 +3,7 @@ package start import ( "context" "fmt" + "github.com/openshift/cluster-bootstrap/pkg/common" "reflect" "strings" "time" @@ -28,7 +29,7 @@ func waitUntilPodsRunning(ctx context.Context, c kubernetes.Interface, pods map[ return fmt.Errorf("error while checking pod status: %v", err) } - UserOutput("All self-hosted control plane components successfully started\n") + common.UserOutput("All self-hosted control plane components successfully started\n") return nil } @@ -96,7 +97,7 @@ func (s *statusController) AllRunningAndReady() (bool, error) { status = string(s.Phase) } - UserOutput("\tPod Status:%24s\t%s\n", p, status) + common.UserOutput("\tPod Status:%24s\t%s\n", p, status) } if s == nil || s.Phase != v1.PodRunning || !s.IsReady { runningAndReady = false diff --git a/vendor/github.com/coreos/ignition/v2/CONTRIBUTING.md b/vendor/github.com/coreos/ignition/v2/CONTRIBUTING.md new file mode 100644 index 000000000..eedf0138d --- /dev/null +++ b/vendor/github.com/coreos/ignition/v2/CONTRIBUTING.md @@ -0,0 +1,53 @@ +# How to Contribute + +CoreOS projects are [Apache 2.0 licensed](LICENSE) and accept contributions via GitHub pull requests. This document outlines some of the conventions on development workflow, commit message formatting, contact points and other resources to make it easier to get your contribution accepted. + +# Certificate of Origin + +By contributing to this project you agree to the Developer Certificate of Origin (DCO). This document was created by the Linux Kernel community and is a simple statement that you, as a contributor, have the legal right to make the contribution. See the [DCO](DCO) file for details. + +## Getting Started + +- Fork the repository on GitHub +- Read the [README](README.md) and [development guide][dev-guide] for build and test instructions +- Play with the project, submit bugs, submit patches! + +## Contribution Flow + +This is a rough outline of what a contributor's workflow looks like: + +- Create a topic branch from where you want to base your work (usually master). +- Make commits of logical units. +- Make sure your commit messages are in the proper format (see below). +- Push your changes to a topic branch in your fork of the repository. +- Make sure the tests pass, and add any new tests as appropriate. +- Submit a pull request to the original repository. + +Thanks for your contributions! + +### Format of the Commit Message + +We follow a rough convention for commit messages that is designed to answer two questions: what changed and why. The subject line should feature the what and the body of the commit should describe the why. + +``` +scripts: add the test-cluster command + +this uses tmux to setup a test cluster that you can easily kill and +start for debugging. + +Fixes #38 +``` + +The format can be described more formally as follows: + +``` +: + + + +