From ccfb65d6c9aff82e287b64da9c7843a780ac7b93 Mon Sep 17 00:00:00 2001 From: Vadim Rutkovsky Date: Wed, 29 Jan 2020 22:08:25 +0100 Subject: [PATCH 1/6] pkg/asset/releaseimage/default: set default release image to 4.4 --- pkg/asset/releaseimage/default.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/asset/releaseimage/default.go b/pkg/asset/releaseimage/default.go index dcf16bd47b4..9da20022a42 100644 --- a/pkg/asset/releaseimage/default.go +++ b/pkg/asset/releaseimage/default.go @@ -20,7 +20,7 @@ import ( var ( // defaultReleaseImageOriginal is the value served when defaultReleaseImagePadded is unmodified. - defaultReleaseImageOriginal = "registry.svc.ci.openshift.org/origin/release:4.3" + defaultReleaseImageOriginal = "registry.svc.ci.openshift.org/origin/release:4.4" // defaultReleaseImagePadded may be replaced in the binary with a pull spec that overrides defaultReleaseImage as // a null-terminated string within the allowed character length. This allows a distributor to override the payload // location without having to rebuild the source. From cc1dde7c7097e20a60700aabc522405f716160e5 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Tue, 28 Jan 2020 10:59:39 -0800 Subject: [PATCH 2/6] data/bootstrap: delay the removal of bootstrap mcs The bootstrap MCS is responsible for serving Ignition configs to the booting control plane machines while we wait for the full control plane (which will host the in-cluster MCS). If we remove this too early, it will prevent the control plane machines from booting and the control plane from starting. The etcd health check, which is the gate before we remove the MCS, reports healthly (since the one-node, bootstrap etcd cluster is actually up) before the control plane machines had a chance to boot. The result is that one or more of the control plane machines get stuck during boot (because Ignition is still trying to fetch a config) and the cluster fails to bootstrap. This moves the bootstrap MCS removal after bootkube has finished, which will ensure that all of the initial manifests have been loaded into the cluster (including the MCO/MCS manifests). --- data/data/bootstrap/files/usr/local/bin/bootkube.sh.template | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index 700f2c7209a..85440df5285 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -397,7 +397,6 @@ done echo "etcd cluster up. Killing etcd certificate signer..." podman rm --force etcd-signer -rm --force /etc/kubernetes/manifests/machineconfigoperator-bootstrap-pod.yaml echo "Starting cluster-bootstrap..." @@ -408,6 +407,8 @@ bootkube_podman_run \ "${CLUSTER_BOOTSTRAP_IMAGE}" \ start --tear-down-early=false --asset-dir=/assets --required-pods="openshift-kube-apiserver/kube-apiserver,openshift-kube-scheduler/openshift-kube-scheduler,openshift-kube-controller-manager/kube-controller-manager,openshift-cluster-version/cluster-version-operator" +rm --force /etc/kubernetes/manifests/machineconfigoperator-bootstrap-pod.yaml + if [ ! -z "$CLUSTER_ETCD_OPERATOR_IMAGE" ] then echo "Waiting for CEO to finish..." From 5cdde43a7fcf5788db70b7e0e01a730b4f16ba63 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Wed, 15 Jan 2020 09:44:19 -0800 Subject: [PATCH 3/6] asset/manifests: add openshift-install configmap This adds support for generating the openshift-install ConfigMap when using a UPI installation flow. Before this, the ConfigMap was generated during the "cluster" target rather than the manifests target, so that we could distinguish between an IPI and a UPI installation. The user had the option of overriding the "invoker" field using the environment variable OPENSHIFT_INSTALL_INVOKER. Even if the environment variable was used, the ConfigMap would not be present in a UPI installation (because it wasn't generated in the "manifests" target). This change causes the installer to generate that ConfigMap in the "manifests" target (in addition to the "cluster" target) when it sees the environment variable. We will make use of this functionality in CI, where we specify the job as the invoker. --- pkg/asset/cluster/tfvars.go | 31 ++--- pkg/asset/manifests/openshift.go | 9 +- .../openshiftinstall/openshiftinstall.go | 115 ++++++++++++++++++ 3 files changed, 140 insertions(+), 15 deletions(-) create mode 100644 pkg/asset/openshiftinstall/openshiftinstall.go diff --git a/pkg/asset/cluster/tfvars.go b/pkg/asset/cluster/tfvars.go index 3351256729b..a831d65141d 100644 --- a/pkg/asset/cluster/tfvars.go +++ b/pkg/asset/cluster/tfvars.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" igntypes "github.com/coreos/ignition/v2/config/v3_0/types" gcpprovider "github.com/openshift/cluster-api-provider-gcp/pkg/apis/gcpprovider/v1beta1" @@ -25,6 +26,7 @@ import ( gcpconfig "github.com/openshift/installer/pkg/asset/installconfig/gcp" ovirtconfig "github.com/openshift/installer/pkg/asset/installconfig/ovirt" "github.com/openshift/installer/pkg/asset/machines" + "github.com/openshift/installer/pkg/asset/openshiftinstall" "github.com/openshift/installer/pkg/asset/rhcos" "github.com/openshift/installer/pkg/tfvars" awstfvars "github.com/openshift/installer/pkg/tfvars/aws" @@ -46,7 +48,6 @@ import ( openstackdefaults "github.com/openshift/installer/pkg/types/openstack/defaults" "github.com/openshift/installer/pkg/types/ovirt" "github.com/openshift/installer/pkg/types/vsphere" - "github.com/openshift/installer/pkg/version" ) const ( @@ -448,26 +449,28 @@ func (t *TerraformVariables) Load(f asset.FileFetcher) (found bool, err error) { // injectInstallInfo adds information about the installer and its invoker as a // ConfigMap to the provided bootstrap Ignition config. func injectInstallInfo(bootstrap []byte) (string, error) { + openshiftInstallPath := filepath.Join("/", "opt", "openshift", openshiftinstall.ConfigPath) config := &igntypes.Config{} if err := json.Unmarshal(bootstrap, &config); err != nil { return "", errors.Wrap(err, "failed to unmarshal bootstrap Ignition config") } - invoker := "user" - if env := os.Getenv("OPENSHIFT_INSTALL_INVOKER"); env != "" { - invoker = env + // If the openshift-install ConfigMap is already present, don't bother + // injecting another. In fact, while it's okay in Ignition v0s2 for a file + // to be defined multiple times (the last occurrence takes precedence), it + // is an error in Ignition v2s3, which is used by OKD. + for _, file := range config.Storage.Files { + if file.Path == openshiftInstallPath { + return string(bootstrap), nil + } + } + + cm, err := openshiftinstall.CreateInstallConfig("user") + if err != nil { + return "", errors.Wrap(err, "failed to generate openshift-install config") } - config.Storage.Files = append(config.Storage.Files, ignition.FileFromString("/opt/openshift/manifests/openshift-install.yml", "root", 0644, fmt.Sprintf(`--- -apiVersion: v1 -kind: ConfigMap -metadata: - name: openshift-install - namespace: openshift-config -data: - version: "%s" - invoker: "%s" -`, version.Raw, invoker))) + config.Storage.Files = append(config.Storage.Files, ignition.FileFromString(openshiftInstallPath, "root", 0644, cm)) ign, err := json.Marshal(config) if err != nil { diff --git a/pkg/asset/manifests/openshift.go b/pkg/asset/manifests/openshift.go index 263008c704b..ccbc9c34ea3 100644 --- a/pkg/asset/manifests/openshift.go +++ b/pkg/asset/manifests/openshift.go @@ -17,6 +17,7 @@ import ( "github.com/openshift/installer/pkg/asset/installconfig/ovirt" "github.com/openshift/installer/pkg/asset/machines" openstackmanifests "github.com/openshift/installer/pkg/asset/manifests/openstack" + "github.com/openshift/installer/pkg/asset/openshiftinstall" osmachine "github.com/openshift/installer/pkg/asset/machines/openstack" "github.com/openshift/installer/pkg/asset/password" @@ -55,6 +56,7 @@ func (o *Openshift) Dependencies() []asset.Asset { &installconfig.InstallConfig{}, &installconfig.ClusterID{}, &password.KubeadminPassword{}, + &openshiftinstall.Config{}, &openshift.CloudCredsSecret{}, &openshift.KubeadminPasswordSecret{}, @@ -69,7 +71,8 @@ func (o *Openshift) Generate(dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} clusterID := &installconfig.ClusterID{} kubeadminPassword := &password.KubeadminPassword{} - dependencies.Get(installConfig, kubeadminPassword, clusterID) + openshiftInstall := &openshiftinstall.Config{} + dependencies.Get(installConfig, kubeadminPassword, clusterID, openshiftInstall) var cloudCreds cloudCredsSecretData platform := installConfig.Config.Platform.Name() switch platform { @@ -215,6 +218,10 @@ func (o *Openshift) Generate(dependencies asset.Parents) error { }) } + if openshiftInstall.File != nil { + o.FileList = append(o.FileList, openshiftInstall.Files()...) + } + asset.SortFiles(o.FileList) return nil diff --git a/pkg/asset/openshiftinstall/openshiftinstall.go b/pkg/asset/openshiftinstall/openshiftinstall.go new file mode 100644 index 00000000000..9ddb2484b58 --- /dev/null +++ b/pkg/asset/openshiftinstall/openshiftinstall.go @@ -0,0 +1,115 @@ +package openshiftinstall + +import ( + "os" + "path/filepath" + + "github.com/ghodss/yaml" + "github.com/pkg/errors" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/version" +) + +var ( + // ConfigPath is the relative path of openshift-install within the asset + // directory. + ConfigPath = filepath.Join("openshift", "openshift-install.yaml") +) + +// Config generates the openshift-install ConfigMap. +type Config struct { + File *asset.File +} + +var _ asset.WritableAsset = (*Config)(nil) + +// Name returns a human friendly name for the asset. +func (*Config) Name() string { + return "OpenShift Install" +} + +// Dependencies returns all of the dependencies directly needed to generate +// the asset. +func (*Config) Dependencies() []asset.Asset { + return []asset.Asset{} +} + +// Generate generates the openshift-install ConfigMap. +func (i *Config) Generate(dependencies asset.Parents) error { + cm, err := CreateInstallConfig("") + if err != nil { + return err + } + + if cm != "" { + i.File = &asset.File{ + Filename: ConfigPath, + Data: []byte(cm), + } + } + + return nil +} + +// Files returns the files generated by the asset. +func (i *Config) Files() []*asset.File { + if i.File != nil { + return []*asset.File{i.File} + } + return []*asset.File{} +} + +// Load loads the already-rendered files back from disk. +func (i *Config) Load(f asset.FileFetcher) (bool, error) { + file, err := f.FetchByName(ConfigPath) + if os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + i.File = file + return true, nil +} + +// CreateInstallConfig creates the openshift-install ConfigMap from the +// OPENSHIFT_INSTALL_INVOKER environment variable, and if not present, from the +// provided default invoker. If both the environment variable and the default +// are the empty string, this returns an empty string (indicting that no +// ConfigMap should be created. This returns an error if the marshalling to +// YAML fails. +func CreateInstallConfig(defaultInvoker string) (string, error) { + var invoker string + if env := os.Getenv("OPENSHIFT_INSTALL_INVOKER"); env != "" { + invoker = env + } else if defaultInvoker != "" { + invoker = defaultInvoker + } else { + return "", nil + } + + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "openshift-config", + Name: "openshift-install", + }, + Data: map[string]string{ + "version": version.Raw, + "invoker": invoker, + }, + } + + cmData, err := yaml.Marshal(cm) + if err != nil { + return "", errors.Wrapf(err, "failed to create install-config ConfigMap") + } + + return string(cmData), nil +} From c91b56c6829dc8dd11dd5433bc076bdcc544a52a Mon Sep 17 00:00:00 2001 From: Vadim Rutkovsky Date: Wed, 29 Jan 2020 14:10:30 +0100 Subject: [PATCH 4/6] data/data/bootstrap/files/usr/local/bin/bootkube.sh: remove runc workaround --- data/data/bootstrap/files/usr/local/bin/bootkube.sh.template | 4 ---- 1 file changed, 4 deletions(-) diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index 85440df5285..1d167577623 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -66,10 +66,6 @@ fi # Run crio-configure after the pivot /usr/local/bin/crio-configure.sh -# Use runc for podman runs -cp /usr/share/containers/libpod.conf /etc/containers/libpod.conf -sed --in-place --expression "s,^runtime *=.*,runtime = \"runc\"," /etc/containers/libpod.conf - mkdir --parents ./{bootstrap-manifests,manifests} if [ ! -f openshift-manifests.done ] From 64642498c906464e6b36379817c2febfa0015329 Mon Sep 17 00:00:00 2001 From: Vadim Rutkovsky Date: Fri, 31 Jan 2020 17:48:00 +0100 Subject: [PATCH 5/6] bootkube.sh: disable zincati before pivoting --- data/data/bootstrap/files/usr/local/bin/bootkube.sh.template | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index 1d167577623..7e4c91c06b1 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -59,6 +59,9 @@ if [ ! -f .pivot-done ]; then chmod +x /usr/local/bin/machine-config-daemon restorecon /usr/local/bin/machine-config-daemon + # Disable zincati + systemctl disable zincati.service + touch .pivot-done /usr/local/bin/machine-config-daemon pivot fi From 88f4b1ca8420b8e4524aa6fba46edf02826e83fc Mon Sep 17 00:00:00 2001 From: Vadim Rutkovsky Date: Sat, 1 Feb 2020 14:12:52 +0100 Subject: [PATCH 6/6] bootstrap: prepull all podman images Make sure all images are pullable before running those. This ensures detached podman containers won't hang is a weird state --- .../files/usr/local/bin/bootkube.sh.template | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index 7e4c91c06b1..ac6ff4c21eb 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -7,6 +7,10 @@ mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests ETCD_ENDPOINTS= +bootkube_podman_pull() { + until podman pull "${@}"; do echo "Retrying pull ${@}"; sleep 5; done +} + bootkube_podman_run() { # we run all commands in the host-network to prevent IP conflicts with # end-user infrastructure. @@ -21,6 +25,9 @@ MACHINE_CONFIG_INFRA_IMAGE=$(image_for pod) KUBE_ETCD_SIGNER_SERVER_IMAGE=$(image_for kube-etcd-signer-server) CLUSTER_ETCD_OPERATOR_IMAGE=$(image_for cluster-etcd-operator || echo "no-ceo-image") + +bootkube_podman_pull "${CLUSTER_ETCD_OPERATOR_IMAGE}" + CLUSTER_ETCD_OPERATOR_MANAGED=${CLUSTER_ETCD_OPERATOR_IMAGE:+$(bootkube_podman_run \ "${CLUSTER_ETCD_OPERATOR_IMAGE}" \ /usr/bin/grep -oP Managed \ @@ -54,6 +61,7 @@ if [ ! -f .pivot-done ]; then echo "DELETE mitigations=auto,nosmt" >> /etc/pivot/kernel-args mkdir --parents bin/ + bootkube_podman_pull "${MACHINE_CONFIG_OPERATOR_IMAGE}" bootkube_podman_run --entrypoint=cat "${MACHINE_CONFIG_OPERATOR_IMAGE}" \ /usr/bin/machine-config-daemon > /usr/local/bin/machine-config-daemon chmod +x /usr/local/bin/machine-config-daemon @@ -84,6 +92,7 @@ then rm --recursive --force cvo-bootstrap + bootkube_podman_pull "${RELEASE_IMAGE_DIGEST}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${RELEASE_IMAGE_DIGEST}" \ @@ -109,6 +118,7 @@ echo "Starting etcd certificate signer..." trap "podman rm --force etcd-signer" ERR +bootkube_podman_pull "${KUBE_ETCD_SIGNER_SERVER_IMAGE}" bootkube_podman_run \ --name etcd-signer \ --detach \ @@ -143,6 +153,7 @@ then if [ ! -f etcd-bootstrap.done ] then echo "Rendering CEO Manifests..." + bootkube_podman_pull "${CLUSTER_ETCD_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${CLUSTER_ETCD_OPERATOR_IMAGE}" \ @@ -183,6 +194,7 @@ then rm --recursive --force config-bootstrap + bootkube_podman_pull "${CONFIG_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${CONFIG_OPERATOR_IMAGE}" \ @@ -202,6 +214,7 @@ then rm --recursive --force kube-apiserver-bootstrap + bootkube_podman_pull "${KUBE_APISERVER_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${KUBE_APISERVER_OPERATOR_IMAGE}" \ @@ -228,6 +241,7 @@ then rm --recursive --force kube-controller-manager-bootstrap + bootkube_podman_pull "${KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE}" \ @@ -251,6 +265,7 @@ then rm --recursive --force kube-scheduler-bootstrap + bootkube_podman_pull "${KUBE_SCHEDULER_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${KUBE_SCHEDULER_OPERATOR_IMAGE}" \ @@ -273,6 +288,7 @@ then rm --recursive --force ingress-operator-bootstrap + bootkube_podman_pull "${INGRESS_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${INGRESS_OPERATOR_IMAGE}" \ @@ -291,6 +307,7 @@ then rm --recursive --force mco-bootstrap + bootkube_podman_pull "${MACHINE_CONFIG_OPERATOR_IMAGE}" bootkube_podman_run \ --user 0 \ --volume "$PWD:/assets:z" \ @@ -359,6 +376,7 @@ then rm --recursive --force cco-bootstrap # shellcheck disable=SC2154 + bootkube_podman_pull "${CLOUD_CREDENTIAL_OPERATOR_IMAGE}" bootkube_podman_run \ --quiet \ --user 0 \ @@ -375,6 +393,7 @@ then fi # Wait for the etcd cluster to come up. +bootkube_podman_pull "${MACHINE_CONFIG_ETCD_IMAGE}" until bootkube_podman_run \ --rm \ --name etcdctl \ @@ -399,6 +418,7 @@ podman rm --force etcd-signer echo "Starting cluster-bootstrap..." +bootkube_podman_pull "${CLUSTER_BOOTSTRAP_IMAGE}" bootkube_podman_run \ --rm \ --volume "$PWD:/assets:z" \ @@ -411,6 +431,7 @@ rm --force /etc/kubernetes/manifests/machineconfigoperator-bootstrap-pod.yaml if [ ! -z "$CLUSTER_ETCD_OPERATOR_IMAGE" ] then echo "Waiting for CEO to finish..." + bootkube_podman_pull "${CLUSTER_ETCD_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${CLUSTER_ETCD_OPERATOR_IMAGE}" \