diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index 700f2c7209a..ac6ff4c21eb 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -7,6 +7,10 @@ mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests ETCD_ENDPOINTS= +bootkube_podman_pull() { + until podman pull "${@}"; do echo "Retrying pull ${@}"; sleep 5; done +} + bootkube_podman_run() { # we run all commands in the host-network to prevent IP conflicts with # end-user infrastructure. @@ -21,6 +25,9 @@ MACHINE_CONFIG_INFRA_IMAGE=$(image_for pod) KUBE_ETCD_SIGNER_SERVER_IMAGE=$(image_for kube-etcd-signer-server) CLUSTER_ETCD_OPERATOR_IMAGE=$(image_for cluster-etcd-operator || echo "no-ceo-image") + +bootkube_podman_pull "${CLUSTER_ETCD_OPERATOR_IMAGE}" + CLUSTER_ETCD_OPERATOR_MANAGED=${CLUSTER_ETCD_OPERATOR_IMAGE:+$(bootkube_podman_run \ "${CLUSTER_ETCD_OPERATOR_IMAGE}" \ /usr/bin/grep -oP Managed \ @@ -54,11 +61,15 @@ if [ ! -f .pivot-done ]; then echo "DELETE mitigations=auto,nosmt" >> /etc/pivot/kernel-args mkdir --parents bin/ + bootkube_podman_pull "${MACHINE_CONFIG_OPERATOR_IMAGE}" bootkube_podman_run --entrypoint=cat "${MACHINE_CONFIG_OPERATOR_IMAGE}" \ /usr/bin/machine-config-daemon > /usr/local/bin/machine-config-daemon chmod +x /usr/local/bin/machine-config-daemon restorecon /usr/local/bin/machine-config-daemon + # Disable zincati + systemctl disable zincati.service + touch .pivot-done /usr/local/bin/machine-config-daemon pivot fi @@ -66,10 +77,6 @@ fi # Run crio-configure after the pivot /usr/local/bin/crio-configure.sh -# Use runc for podman runs -cp /usr/share/containers/libpod.conf /etc/containers/libpod.conf -sed --in-place --expression "s,^runtime *=.*,runtime = \"runc\"," /etc/containers/libpod.conf - mkdir --parents ./{bootstrap-manifests,manifests} if [ ! -f openshift-manifests.done ] @@ -85,6 +92,7 @@ then rm --recursive --force cvo-bootstrap + bootkube_podman_pull "${RELEASE_IMAGE_DIGEST}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${RELEASE_IMAGE_DIGEST}" \ @@ -110,6 +118,7 @@ echo "Starting etcd certificate signer..." trap "podman rm --force etcd-signer" ERR +bootkube_podman_pull "${KUBE_ETCD_SIGNER_SERVER_IMAGE}" bootkube_podman_run \ --name etcd-signer \ --detach \ @@ -144,6 +153,7 @@ then if [ ! -f etcd-bootstrap.done ] then echo "Rendering CEO Manifests..." + bootkube_podman_pull "${CLUSTER_ETCD_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${CLUSTER_ETCD_OPERATOR_IMAGE}" \ @@ -184,6 +194,7 @@ then rm --recursive --force config-bootstrap + bootkube_podman_pull "${CONFIG_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${CONFIG_OPERATOR_IMAGE}" \ @@ -203,6 +214,7 @@ then rm --recursive --force kube-apiserver-bootstrap + bootkube_podman_pull "${KUBE_APISERVER_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${KUBE_APISERVER_OPERATOR_IMAGE}" \ @@ -229,6 +241,7 @@ then rm --recursive --force kube-controller-manager-bootstrap + bootkube_podman_pull "${KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE}" \ @@ -252,6 +265,7 @@ then rm --recursive --force kube-scheduler-bootstrap + bootkube_podman_pull "${KUBE_SCHEDULER_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${KUBE_SCHEDULER_OPERATOR_IMAGE}" \ @@ -274,6 +288,7 @@ then rm --recursive --force ingress-operator-bootstrap + bootkube_podman_pull "${INGRESS_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${INGRESS_OPERATOR_IMAGE}" \ @@ -292,6 +307,7 @@ then rm --recursive --force mco-bootstrap + bootkube_podman_pull "${MACHINE_CONFIG_OPERATOR_IMAGE}" bootkube_podman_run \ --user 0 \ --volume "$PWD:/assets:z" \ @@ -360,6 +376,7 @@ then rm --recursive --force cco-bootstrap # shellcheck disable=SC2154 + bootkube_podman_pull "${CLOUD_CREDENTIAL_OPERATOR_IMAGE}" bootkube_podman_run \ --quiet \ --user 0 \ @@ -376,6 +393,7 @@ then fi # Wait for the etcd cluster to come up. +bootkube_podman_pull "${MACHINE_CONFIG_ETCD_IMAGE}" until bootkube_podman_run \ --rm \ --name etcdctl \ @@ -397,10 +415,10 @@ done echo "etcd cluster up. Killing etcd certificate signer..." podman rm --force etcd-signer -rm --force /etc/kubernetes/manifests/machineconfigoperator-bootstrap-pod.yaml echo "Starting cluster-bootstrap..." +bootkube_podman_pull "${CLUSTER_BOOTSTRAP_IMAGE}" bootkube_podman_run \ --rm \ --volume "$PWD:/assets:z" \ @@ -408,9 +426,12 @@ bootkube_podman_run \ "${CLUSTER_BOOTSTRAP_IMAGE}" \ start --tear-down-early=false --asset-dir=/assets --required-pods="openshift-kube-apiserver/kube-apiserver,openshift-kube-scheduler/openshift-kube-scheduler,openshift-kube-controller-manager/kube-controller-manager,openshift-cluster-version/cluster-version-operator" +rm --force /etc/kubernetes/manifests/machineconfigoperator-bootstrap-pod.yaml + if [ ! -z "$CLUSTER_ETCD_OPERATOR_IMAGE" ] then echo "Waiting for CEO to finish..." + bootkube_podman_pull "${CLUSTER_ETCD_OPERATOR_IMAGE}" bootkube_podman_run \ --volume "$PWD:/assets:z" \ "${CLUSTER_ETCD_OPERATOR_IMAGE}" \ diff --git a/pkg/asset/cluster/tfvars.go b/pkg/asset/cluster/tfvars.go index 3351256729b..a831d65141d 100644 --- a/pkg/asset/cluster/tfvars.go +++ b/pkg/asset/cluster/tfvars.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" igntypes "github.com/coreos/ignition/v2/config/v3_0/types" gcpprovider "github.com/openshift/cluster-api-provider-gcp/pkg/apis/gcpprovider/v1beta1" @@ -25,6 +26,7 @@ import ( gcpconfig "github.com/openshift/installer/pkg/asset/installconfig/gcp" ovirtconfig "github.com/openshift/installer/pkg/asset/installconfig/ovirt" "github.com/openshift/installer/pkg/asset/machines" + "github.com/openshift/installer/pkg/asset/openshiftinstall" "github.com/openshift/installer/pkg/asset/rhcos" "github.com/openshift/installer/pkg/tfvars" awstfvars "github.com/openshift/installer/pkg/tfvars/aws" @@ -46,7 +48,6 @@ import ( openstackdefaults "github.com/openshift/installer/pkg/types/openstack/defaults" "github.com/openshift/installer/pkg/types/ovirt" "github.com/openshift/installer/pkg/types/vsphere" - "github.com/openshift/installer/pkg/version" ) const ( @@ -448,26 +449,28 @@ func (t *TerraformVariables) Load(f asset.FileFetcher) (found bool, err error) { // injectInstallInfo adds information about the installer and its invoker as a // ConfigMap to the provided bootstrap Ignition config. func injectInstallInfo(bootstrap []byte) (string, error) { + openshiftInstallPath := filepath.Join("/", "opt", "openshift", openshiftinstall.ConfigPath) config := &igntypes.Config{} if err := json.Unmarshal(bootstrap, &config); err != nil { return "", errors.Wrap(err, "failed to unmarshal bootstrap Ignition config") } - invoker := "user" - if env := os.Getenv("OPENSHIFT_INSTALL_INVOKER"); env != "" { - invoker = env + // If the openshift-install ConfigMap is already present, don't bother + // injecting another. In fact, while it's okay in Ignition v0s2 for a file + // to be defined multiple times (the last occurrence takes precedence), it + // is an error in Ignition v2s3, which is used by OKD. + for _, file := range config.Storage.Files { + if file.Path == openshiftInstallPath { + return string(bootstrap), nil + } + } + + cm, err := openshiftinstall.CreateInstallConfig("user") + if err != nil { + return "", errors.Wrap(err, "failed to generate openshift-install config") } - config.Storage.Files = append(config.Storage.Files, ignition.FileFromString("/opt/openshift/manifests/openshift-install.yml", "root", 0644, fmt.Sprintf(`--- -apiVersion: v1 -kind: ConfigMap -metadata: - name: openshift-install - namespace: openshift-config -data: - version: "%s" - invoker: "%s" -`, version.Raw, invoker))) + config.Storage.Files = append(config.Storage.Files, ignition.FileFromString(openshiftInstallPath, "root", 0644, cm)) ign, err := json.Marshal(config) if err != nil { diff --git a/pkg/asset/manifests/openshift.go b/pkg/asset/manifests/openshift.go index 263008c704b..ccbc9c34ea3 100644 --- a/pkg/asset/manifests/openshift.go +++ b/pkg/asset/manifests/openshift.go @@ -17,6 +17,7 @@ import ( "github.com/openshift/installer/pkg/asset/installconfig/ovirt" "github.com/openshift/installer/pkg/asset/machines" openstackmanifests "github.com/openshift/installer/pkg/asset/manifests/openstack" + "github.com/openshift/installer/pkg/asset/openshiftinstall" osmachine "github.com/openshift/installer/pkg/asset/machines/openstack" "github.com/openshift/installer/pkg/asset/password" @@ -55,6 +56,7 @@ func (o *Openshift) Dependencies() []asset.Asset { &installconfig.InstallConfig{}, &installconfig.ClusterID{}, &password.KubeadminPassword{}, + &openshiftinstall.Config{}, &openshift.CloudCredsSecret{}, &openshift.KubeadminPasswordSecret{}, @@ -69,7 +71,8 @@ func (o *Openshift) Generate(dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} clusterID := &installconfig.ClusterID{} kubeadminPassword := &password.KubeadminPassword{} - dependencies.Get(installConfig, kubeadminPassword, clusterID) + openshiftInstall := &openshiftinstall.Config{} + dependencies.Get(installConfig, kubeadminPassword, clusterID, openshiftInstall) var cloudCreds cloudCredsSecretData platform := installConfig.Config.Platform.Name() switch platform { @@ -215,6 +218,10 @@ func (o *Openshift) Generate(dependencies asset.Parents) error { }) } + if openshiftInstall.File != nil { + o.FileList = append(o.FileList, openshiftInstall.Files()...) + } + asset.SortFiles(o.FileList) return nil diff --git a/pkg/asset/openshiftinstall/openshiftinstall.go b/pkg/asset/openshiftinstall/openshiftinstall.go new file mode 100644 index 00000000000..9ddb2484b58 --- /dev/null +++ b/pkg/asset/openshiftinstall/openshiftinstall.go @@ -0,0 +1,115 @@ +package openshiftinstall + +import ( + "os" + "path/filepath" + + "github.com/ghodss/yaml" + "github.com/pkg/errors" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/version" +) + +var ( + // ConfigPath is the relative path of openshift-install within the asset + // directory. + ConfigPath = filepath.Join("openshift", "openshift-install.yaml") +) + +// Config generates the openshift-install ConfigMap. +type Config struct { + File *asset.File +} + +var _ asset.WritableAsset = (*Config)(nil) + +// Name returns a human friendly name for the asset. +func (*Config) Name() string { + return "OpenShift Install" +} + +// Dependencies returns all of the dependencies directly needed to generate +// the asset. +func (*Config) Dependencies() []asset.Asset { + return []asset.Asset{} +} + +// Generate generates the openshift-install ConfigMap. +func (i *Config) Generate(dependencies asset.Parents) error { + cm, err := CreateInstallConfig("") + if err != nil { + return err + } + + if cm != "" { + i.File = &asset.File{ + Filename: ConfigPath, + Data: []byte(cm), + } + } + + return nil +} + +// Files returns the files generated by the asset. +func (i *Config) Files() []*asset.File { + if i.File != nil { + return []*asset.File{i.File} + } + return []*asset.File{} +} + +// Load loads the already-rendered files back from disk. +func (i *Config) Load(f asset.FileFetcher) (bool, error) { + file, err := f.FetchByName(ConfigPath) + if os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + i.File = file + return true, nil +} + +// CreateInstallConfig creates the openshift-install ConfigMap from the +// OPENSHIFT_INSTALL_INVOKER environment variable, and if not present, from the +// provided default invoker. If both the environment variable and the default +// are the empty string, this returns an empty string (indicting that no +// ConfigMap should be created. This returns an error if the marshalling to +// YAML fails. +func CreateInstallConfig(defaultInvoker string) (string, error) { + var invoker string + if env := os.Getenv("OPENSHIFT_INSTALL_INVOKER"); env != "" { + invoker = env + } else if defaultInvoker != "" { + invoker = defaultInvoker + } else { + return "", nil + } + + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "openshift-config", + Name: "openshift-install", + }, + Data: map[string]string{ + "version": version.Raw, + "invoker": invoker, + }, + } + + cmData, err := yaml.Marshal(cm) + if err != nil { + return "", errors.Wrapf(err, "failed to create install-config ConfigMap") + } + + return string(cmData), nil +} diff --git a/pkg/asset/releaseimage/default.go b/pkg/asset/releaseimage/default.go index dcf16bd47b4..9da20022a42 100644 --- a/pkg/asset/releaseimage/default.go +++ b/pkg/asset/releaseimage/default.go @@ -20,7 +20,7 @@ import ( var ( // defaultReleaseImageOriginal is the value served when defaultReleaseImagePadded is unmodified. - defaultReleaseImageOriginal = "registry.svc.ci.openshift.org/origin/release:4.3" + defaultReleaseImageOriginal = "registry.svc.ci.openshift.org/origin/release:4.4" // defaultReleaseImagePadded may be replaced in the binary with a pull spec that overrides defaultReleaseImage as // a null-terminated string within the allowed character length. This allows a distributor to override the payload // location without having to rebuild the source.