diff --git a/cmd/openshift-install/main.go b/cmd/openshift-install/main.go index 8cd6afffd47..07e32e1575e 100644 --- a/cmd/openshift-install/main.go +++ b/cmd/openshift-install/main.go @@ -11,7 +11,8 @@ import ( ) var ( - installConfigCommand = kingpin.Command("install-config", "Generate the Install Config asset") + installConfigCommand = kingpin.Command("install-config", "Generate the Install Config asset") + ignitionConfigsCommand = kingpin.Command("ignition-configs", "Generate the Ignition Config assets") dirFlag = kingpin.Flag("dir", "assets directory").Default(".").String() logLevel = kingpin.Flag("log-level", "log level (e.g. \"debug\")").Default("info").Enum("debug", "info", "warn", "error", "fatal", "panic") @@ -22,11 +23,16 @@ func main() { assetStock := stock.EstablishStock(*dirFlag) - var targetAsset asset.Asset - + var targetAssets []asset.Asset switch command { case installConfigCommand.FullCommand(): - targetAsset = assetStock.InstallConfig() + targetAssets = []asset.Asset{assetStock.InstallConfig()} + case ignitionConfigsCommand.FullCommand(): + targetAssets = []asset.Asset{ + assetStock.BootstrapIgnition(), + assetStock.MasterIgnition(), + assetStock.WorkerIgnition(), + } } l, err := log.ParseLevel(*logLevel) @@ -37,14 +43,16 @@ func main() { log.SetLevel(l) assetStore := &asset.StoreImpl{} - st, err := assetStore.Fetch(targetAsset) - if err != nil { - log.Fatalf("failed to generate asset: %v", err) - os.Exit(1) - } - - if err := st.PersistToFile(); err != nil { - log.Fatalf("failed to write target to disk: %v", err) - os.Exit(1) + for _, asset := range targetAssets { + st, err := assetStore.Fetch(asset) + if err != nil { + log.Fatalf("failed to generate asset: %v", err) + os.Exit(1) + } + + if err := st.PersistToFile(); err != nil { + log.Fatalf("failed to write target to disk: %v", err) + os.Exit(1) + } } } diff --git a/installer/pkg/config-generator/BUILD.bazel b/installer/pkg/config-generator/BUILD.bazel index 689171cbb38..ae939e9bbf6 100644 --- a/installer/pkg/config-generator/BUILD.bazel +++ b/installer/pkg/config-generator/BUILD.bazel @@ -10,11 +10,11 @@ go_library( importpath = "github.com/openshift/installer/installer/pkg/config-generator", visibility = ["//visibility:public"], deps = [ - "//pkg/rhcos:go_default_library", "//installer/pkg/config:go_default_library", "//installer/pkg/copy:go_default_library", "//pkg/asset/tls:go_default_library", "//pkg/ipnet:go_default_library", + "//pkg/rhcos:go_default_library", "//pkg/types:go_default_library", "//vendor/github.com/apparentlymart/go-cidr/cidr:go_default_library", "//vendor/github.com/coreos/ignition/config/v2_2:go_default_library", diff --git a/pkg/asset/ignition/BUILD.bazel b/pkg/asset/ignition/BUILD.bazel new file mode 100644 index 00000000000..d3691413867 --- /dev/null +++ b/pkg/asset/ignition/BUILD.bazel @@ -0,0 +1,44 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "bootstrap.go", + "doc.go", + "master.go", + "node.go", + "stock.go", + "worker.go", + ], + importpath = "github.com/openshift/installer/pkg/asset/ignition", + visibility = ["//visibility:public"], + deps = [ + "//pkg/asset:go_default_library", + "//pkg/asset/ignition/templates:go_default_library", + "//pkg/asset/installconfig:go_default_library", + "//pkg/asset/kubeconfig:go_default_library", + "//pkg/asset/tls:go_default_library", + "//pkg/types:go_default_library", + "//vendor/github.com/coreos/ignition/config/util:go_default_library", + "//vendor/github.com/coreos/ignition/config/v2_2/types:go_default_library", + "//vendor/github.com/vincent-petithory/dataurl:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "bootstrap_test.go", + "master_test.go", + "testasset_test.go", + "testutils_test.go", + "worker_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//pkg/asset:go_default_library", + "//pkg/asset/ignition/templates:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", + "//vendor/github.com/vincent-petithory/dataurl:go_default_library", + ], +) diff --git a/pkg/asset/ignition/bootstrap.go b/pkg/asset/ignition/bootstrap.go new file mode 100644 index 00000000000..0db038d9477 --- /dev/null +++ b/pkg/asset/ignition/bootstrap.go @@ -0,0 +1,294 @@ +package ignition + +import ( + "bytes" + "encoding/json" + "fmt" + "path" + "path/filepath" + "strings" + "text/template" + + "github.com/coreos/ignition/config/util" + ignition "github.com/coreos/ignition/config/v2_2/types" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/ignition/templates" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/kubeconfig" + "github.com/openshift/installer/pkg/asset/tls" + "github.com/openshift/installer/pkg/types" +) + +const ( + // tlsCertDirectory is the directory on the bootstrap node to place the TLS + // assets. + tlsCertDirectory = "/opt/tectonic/tls" +) + +// bootstrapTemplateData is the data to use to replace values in bootstrap +// template files. +type bootstrapTemplateData struct { + ClusterDNSIP string + CloudProvider string + CloudProviderConfig string + DebugConfig string + KubeCoreRenderImage string + MachineConfigOperatorImage string + EtcdCertSignerImage string + EtcdctlImage string + BootkubeImage string + HyperkubeImage string + EtcdCluster string +} + +// bootstrap is an asset that generates the ignition config for bootstrap nodes. +type bootstrap struct { + directory string + installConfig asset.Asset + rootCA asset.Asset + etcdCA asset.Asset + ingressCertKey asset.Asset + kubeCA asset.Asset + aggregatorCA asset.Asset + serviceServingCA asset.Asset + clusterAPIServerCertKey asset.Asset + etcdClientCertKey asset.Asset + apiServerCertKey asset.Asset + openshiftAPIServerCertKey asset.Asset + apiServerProxyCertKey asset.Asset + adminCertKey asset.Asset + kubeletCertKey asset.Asset + tncCertKey asset.Asset + serviceAccountKeyPair asset.Asset + kubeconfig asset.Asset + kubeconfigKubelet asset.Asset +} + +var _ asset.Asset = (*bootstrap)(nil) + +// newBootstrap creates a new bootstrap asset. +func newBootstrap( + directory string, + installConfigStock installconfig.Stock, + tlsStock tls.Stock, + kubeconfigStock kubeconfig.Stock, +) *bootstrap { + return &bootstrap{ + directory: directory, + installConfig: installConfigStock.InstallConfig(), + rootCA: tlsStock.RootCA(), + etcdCA: tlsStock.EtcdCA(), + ingressCertKey: tlsStock.IngressCertKey(), + kubeCA: tlsStock.KubeCA(), + aggregatorCA: tlsStock.AggregatorCA(), + serviceServingCA: tlsStock.ServiceServingCA(), + clusterAPIServerCertKey: tlsStock.ClusterAPIServerCertKey(), + etcdClientCertKey: tlsStock.EtcdClientCertKey(), + apiServerCertKey: tlsStock.APIServerCertKey(), + openshiftAPIServerCertKey: tlsStock.OpenshiftAPIServerCertKey(), + apiServerProxyCertKey: tlsStock.APIServerProxyCertKey(), + adminCertKey: tlsStock.AdminCertKey(), + kubeletCertKey: tlsStock.KubeletCertKey(), + tncCertKey: tlsStock.TNCCertKey(), + serviceAccountKeyPair: tlsStock.ServiceAccountKeyPair(), + kubeconfig: kubeconfigStock.KubeconfigAdmin(), + kubeconfigKubelet: kubeconfigStock.KubeconfigKubelet(), + } +} + +// Dependencies returns the assets on which the bootstrap asset depends. +func (a *bootstrap) Dependencies() []asset.Asset { + return []asset.Asset{ + a.installConfig, + a.rootCA, + a.etcdCA, + a.ingressCertKey, + a.kubeCA, + a.aggregatorCA, + a.serviceServingCA, + a.clusterAPIServerCertKey, + a.etcdClientCertKey, + a.apiServerCertKey, + a.openshiftAPIServerCertKey, + a.apiServerProxyCertKey, + a.adminCertKey, + a.kubeletCertKey, + a.tncCertKey, + a.serviceAccountKeyPair, + a.kubeconfig, + a.kubeconfigKubelet, + } +} + +// Generate generates the ignition config for the bootstrap asset. +func (a *bootstrap) Generate(dependencies map[asset.Asset]*asset.State) (*asset.State, error) { + installConfig, err := installconfig.GetInstallConfig(a.installConfig, dependencies) + if err != nil { + return nil, err + } + + templateData, err := a.getTemplateData(installConfig) + if err != nil { + return nil, err + } + + config := ignition.Config{} + + a.addBootstrapConfigFiles(&config, dependencies) + a.addBootstrapCertFiles(&config, dependencies) + a.addBootkubeFiles(&config, dependencies, templateData) + a.addTectonicFiles(&config, dependencies, templateData) + a.addTLSCertFiles(&config, dependencies) + + config.Systemd.Units = append( + config.Systemd.Units, + ignition.Unit{Name: "bootkube.service", Contents: templates.BootkubeSystemdContents}, + ignition.Unit{Name: "tectonic.service", Contents: templates.TectonicSystemdContents, Enabled: util.BoolToPtr(true)}, + ignition.Unit{Name: "kubelet.service", Contents: string(applyTemplateData(templates.KubeletSystemdContents, templateData)), Enabled: util.BoolToPtr(true)}, + ) + + config.Passwd.Users = append( + config.Passwd.Users, + ignition.PasswdUser{Name: "core", SSHAuthorizedKeys: []ignition.SSHAuthorizedKey{ignition.SSHAuthorizedKey(installConfig.Admin.SSHKey)}}, + ) + + data, err := json.Marshal(config) + if err != nil { + return nil, err + } + + return &asset.State{ + Contents: []asset.Content{{ + Name: filepath.Join(a.directory, "bootstrap.ign"), + Data: data, + }}, + }, nil +} + +// getTemplateData returns the data to use to execute bootstrap templates. +func (a *bootstrap) getTemplateData(installConfig *types.InstallConfig) (*bootstrapTemplateData, error) { + clusterDNSIP, err := installconfig.ClusterDNSIP(installConfig) + if err != nil { + return nil, err + } + etcdEndpoints := make([]string, masterCount(installConfig)) + for i := range etcdEndpoints { + etcdEndpoints[i] = fmt.Sprintf("https://%s-etcd-%d.%s:2379", installConfig.Name, i, installConfig.BaseDomain) + } + return &bootstrapTemplateData{ + ClusterDNSIP: clusterDNSIP, + CloudProvider: getCloudProvider(installConfig), + CloudProviderConfig: getCloudProviderConfig(installConfig), + DebugConfig: "", + KubeCoreRenderImage: "quay.io/coreos/kube-core-renderer-dev:436b1b4395ae54d866edc88864c9b01797cebac1", + MachineConfigOperatorImage: "docker.io/openshift/origin-machine-config-operator:v4.0.0", + EtcdCertSignerImage: "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6", + EtcdctlImage: "quay.io/coreos/etcd:v3.2.14", + BootkubeImage: "quay.io/coreos/bootkube:v0.10.0", + HyperkubeImage: "openshift/origin-node:latest", + EtcdCluster: strings.Join(etcdEndpoints, ","), + }, nil +} + +func (a *bootstrap) addBootstrapConfigFiles(config *ignition.Config, dependencies map[asset.Asset]*asset.State) { + // TODO (staebler) - missing the following from assets step + // /opt/tectonic/manifests/cluster-config.yaml + // /opt/tectonic/tectonic/cluster-config.yaml + // /opt/tectonic/tnco-config.yaml + // /opt/tectonic/kco-config.yaml + // /etc/kubernetes/kubeconfig + // /var/lib/kubelet/kubeconfig +} + +func (a *bootstrap) addBootstrapCertFiles(config *ignition.Config, dependencies map[asset.Asset]*asset.State) { + config.Storage.Files = append( + config.Storage.Files, + fileFromAsset("/etc/ssl/etcd/ca.crt", 0444, dependencies[a.etcdCA], keyCertAssetCrtIndex), + fileFromAsset("/etc/ssl/etcd/root-ca.crt", 0444, dependencies[a.rootCA], keyCertAssetCrtIndex), + + // ssl certs + fileFromAsset("/etc/ssl/certs/root_ca.pem", 0444, dependencies[a.rootCA], keyCertAssetKeyIndex), + fileFromAsset("/etc/ssl/certs/ingress_ca.pem", 0444, dependencies[a.ingressCertKey], keyCertAssetKeyIndex), + fileFromAsset("/etc/ssl/certs/etcd_ca.pem", 0444, dependencies[a.etcdCA], keyCertAssetKeyIndex), + ) +} + +func (a *bootstrap) addBootkubeFiles(config *ignition.Config, dependencies map[asset.Asset]*asset.State, templateData *bootstrapTemplateData) { + // TODO (staebler) - missing manifests from bootkube module + config.Storage.Files = append( + config.Storage.Files, + fileFromAsset("/opt/tectonic/auth/kubeconfig", 0400, dependencies[a.kubeconfig], 0), + fileFromAsset("/opt/tectonic/auth/kubeconfig-kubelet", 0400, dependencies[a.kubeconfigKubelet], 0), + fileFromBytes("/opt/tectonic/bootkube.sh", 0555, applyTemplateData(templates.BootkubeShFileContents, templateData)), + ) +} + +func (a *bootstrap) addTectonicFiles(config *ignition.Config, dependencies map[asset.Asset]*asset.State, templateData *bootstrapTemplateData) { + // TODO (staebler) - missing manifests from tectonic module + config.Storage.Files = append( + config.Storage.Files, + fileFromBytes("/opt/tectonic/tectonic.sh", 0555, applyTemplateData(templates.TectonicShFileContents, templateData)), + ) +} + +func (a *bootstrap) addTLSCertFiles(config *ignition.Config, dependencies map[asset.Asset]*asset.State) { + for _, pair := range []struct { + key string + crt string + state *asset.State + }{ + {"", "root-ca.crt", dependencies[a.rootCA]}, + {"kube-ca.key", "kube-ca.crt", dependencies[a.kubeCA]}, + {"aggregator-ca.key", "aggregator-ca.crt", dependencies[a.aggregatorCA]}, + {"service-serving-ca.key", "service-serving-ca.crt", dependencies[a.serviceServingCA]}, + {"etcd-client-ca.key", "etcd-client-ca.crt", dependencies[a.etcdCA]}, + {"cluster-apiserver-ca.key", "cluster-apiserver-ca.crt", dependencies[a.clusterAPIServerCertKey]}, + + // etcd cert + {"etcd-client.key", "etcd-client.crt", dependencies[a.etcdClientCertKey]}, + + // kube certs + {"apiserver.key", "apiserver.crt", dependencies[a.apiServerCertKey]}, + {"openshift-apiserver.key", "openshift-apiserver.crt", dependencies[a.openshiftAPIServerCertKey]}, + {"apiserver-proxy.key", "apiserver-proxy.crt", dependencies[a.apiServerProxyCertKey]}, + {"admin.key", "admin.crt", dependencies[a.adminCertKey]}, + {"kubelet.key", "kubelet.crt", dependencies[a.kubeletCertKey]}, + + // tnc cert + {"tnc.key", "tnc.crt", dependencies[a.tncCertKey]}, + + // service account cert + {"service-account.key", "service-account.crt", dependencies[a.serviceAccountKeyPair]}, + } { + if pair.key != "" { + config.Storage.Files = append(config.Storage.Files, fileFromAsset(path.Join(tlsCertDirectory, pair.key), 0600, pair.state, keyCertAssetKeyIndex)) + } + if pair.crt != "" { + config.Storage.Files = append(config.Storage.Files, fileFromAsset(path.Join(tlsCertDirectory, pair.crt), 0644, pair.state, keyCertAssetCrtIndex)) + } + } +} + +func getCloudProvider(installConfig *types.InstallConfig) string { + if installConfig.AWS != nil { + return "aws" + } + return "" +} + +func getCloudProviderConfig(installConfig *types.InstallConfig) string { + return "" +} + +func applyTemplateData(templateString string, templateData interface{}) []byte { + t, err := template.New("").Parse(templateString) + if err != nil { + panic(err) + } + buf := &bytes.Buffer{} + if err := t.Execute(buf, &templateData); err != nil { + panic(err) + } + return buf.Bytes() +} diff --git a/pkg/asset/ignition/bootstrap_test.go b/pkg/asset/ignition/bootstrap_test.go new file mode 100644 index 00000000000..fa28fae46d3 --- /dev/null +++ b/pkg/asset/ignition/bootstrap_test.go @@ -0,0 +1,538 @@ +package ignition + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/ignition/templates" +) + +// TestBootstrapGenerate tests generating the bootstrap asset. +func TestBootstrapGenerate(t *testing.T) { + installConfig := ` +metadata: + name: test-cluster +admin: + email: test-admin-email + sshKey: test-admin-ssh-key +baseDomain: test-domain +networking: + ServiceCIDR: 10.0.1.0/24 +platform: + aws: + region: us-east +machines: +- name: master + replicas: 3 +` + installConfigAsset := &testAsset{"install-config"} + rootCAAsset := &testAsset{"rootCA"} + etcdCAAsset := &testAsset{"etcdCA"} + ingressCertKeyAsset := &testAsset{"ingress-ca"} + kubeCAAsset := &testAsset{"kubeCA"} + aggregatorCAAsset := &testAsset{"aggregator-ca"} + serviceServingCAAsset := &testAsset{"service-serving-ca"} + clusterAPIServerCertKeyAsset := &testAsset{"cluster-apiserver-ca"} + etcdClientCertKeyAsset := &testAsset{"etcd-client-ca"} + apiServerCertKeyAsset := &testAsset{"apiserver-ca"} + openshiftAPIServerCertKeyAsset := &testAsset{"openshift-apiserver-ca"} + apiServerProxyCertKeyAsset := &testAsset{"apiserver-proxy-ca"} + adminCertKeyAsset := &testAsset{"admin-ca"} + kubeletCertKeyAsset := &testAsset{"kubelet-ca"} + tncCertKeyAsset := &testAsset{"tnc-ca"} + serviceAccountKeyPairAsset := &testAsset{"service-account-ca"} + kubeconfigAsset := &testAsset{"kubeconfig"} + kubeconfigKubeletAsset := &testAsset{"kubeconfig-kubelet"} + bootstrap := &bootstrap{ + directory: "test-directory", + installConfig: installConfigAsset, + rootCA: rootCAAsset, + etcdCA: etcdCAAsset, + ingressCertKey: ingressCertKeyAsset, + kubeCA: kubeCAAsset, + aggregatorCA: aggregatorCAAsset, + serviceServingCA: serviceServingCAAsset, + clusterAPIServerCertKey: clusterAPIServerCertKeyAsset, + etcdClientCertKey: etcdClientCertKeyAsset, + apiServerCertKey: apiServerCertKeyAsset, + openshiftAPIServerCertKey: openshiftAPIServerCertKeyAsset, + apiServerProxyCertKey: apiServerProxyCertKeyAsset, + adminCertKey: adminCertKeyAsset, + kubeletCertKey: kubeletCertKeyAsset, + tncCertKey: tncCertKeyAsset, + serviceAccountKeyPair: serviceAccountKeyPairAsset, + kubeconfig: kubeconfigAsset, + kubeconfigKubelet: kubeconfigKubeletAsset, + } + dependencies := map[asset.Asset]*asset.State{ + installConfigAsset: stateWithContentsData(installConfig), + rootCAAsset: stateWithContentsData("test-rootCA-priv", "test-rootCA-pub"), + etcdCAAsset: stateWithContentsData("test-etcdCA-priv", "test-etcdCA-pub"), + ingressCertKeyAsset: stateWithContentsData("test-ingress-ca-priv", "test-ingress-ca-pub"), + kubeCAAsset: stateWithContentsData("test-kubeCA-priv", "test-kubeCA-pub"), + aggregatorCAAsset: stateWithContentsData("test-aggregator-ca-priv", "test-aggregator-ca-pub"), + serviceServingCAAsset: stateWithContentsData("test-service-serving-ca-priv", "test-service-serving-ca-pub"), + clusterAPIServerCertKeyAsset: stateWithContentsData("test-cluster-apiserver-cert-priv", "test-cluster-apiserver-cert-pub"), + etcdClientCertKeyAsset: stateWithContentsData("test-etcd-client-cert-priv", "test-etcd-client-cert-pub"), + apiServerCertKeyAsset: stateWithContentsData("test-apiserver-cert-priv", "test-apiserver-cert-pub"), + openshiftAPIServerCertKeyAsset: stateWithContentsData("test-openshift-apiserver-cert-priv", "test-openshift-apiserver-cert-pub"), + apiServerProxyCertKeyAsset: stateWithContentsData("test-apiserver-proxy-cert-priv", "test-apiserver-proxy-cert-pub"), + adminCertKeyAsset: stateWithContentsData("test-admin-cert-priv", "test-admin-cert-pub"), + kubeletCertKeyAsset: stateWithContentsData("test-kubelet-cert-priv", "test-kubelet-cert-pub"), + tncCertKeyAsset: stateWithContentsData("test-tnc-cert-priv", "test-tnc-cert-pub"), + serviceAccountKeyPairAsset: stateWithContentsData("test-service-account-cert-priv", "test-service-account-cert-pub"), + kubeconfigAsset: stateWithContentsData("test-kubeconfig"), + kubeconfigKubeletAsset: stateWithContentsData("test-kubeconfig-kubelet"), + } + bootstrapState, err := bootstrap.Generate(dependencies) + assert.NoError(t, err, "unexpected error generating bootstrap asset") + assert.Equal(t, 1, len(bootstrapState.Contents), "unexpected number of contents in bootstrap state") + assert.Equal(t, "test-directory/bootstrap.ign", bootstrapState.Contents[0].Name, "unexpected name for bootstrap ignition config") + + assertFilesInIgnitionConfig( + t, + bootstrapState.Contents[0].Data, + fileAssertion{ + path: "/etc/ssl/etcd/ca.crt", + data: "test-etcdCA-pub", + }, + fileAssertion{ + path: "/etc/ssl/etcd/root-ca.crt", + data: "test-rootCA-pub", + }, + fileAssertion{ + path: "/etc/ssl/certs/root_ca.pem", + data: "test-rootCA-priv", + }, + fileAssertion{ + path: "/etc/ssl/certs/ingress_ca.pem", + data: "test-ingress-ca-priv", + }, + fileAssertion{ + path: "/etc/ssl/certs/etcd_ca.pem", + data: "test-etcdCA-priv", + }, + fileAssertion{ + path: "/opt/tectonic/auth/kubeconfig", + data: "test-kubeconfig", + }, + fileAssertion{ + path: "/opt/tectonic/auth/kubeconfig-kubelet", + data: "test-kubeconfig-kubelet", + }, + fileAssertion{ + path: "/opt/tectonic/bootkube.sh", + data: expectedBootkubeSh, + }, + fileAssertion{ + path: "/opt/tectonic/tectonic.sh", + data: expectedTectonicSh, + }, + fileAssertion{ + path: "/opt/tectonic/tls/root-ca.crt", + data: "test-rootCA-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/kube-ca.key", + data: "test-kubeCA-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/kube-ca.crt", + data: "test-kubeCA-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/aggregator-ca.key", + data: "test-aggregator-ca-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/aggregator-ca.crt", + data: "test-aggregator-ca-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/service-serving-ca.key", + data: "test-service-serving-ca-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/service-serving-ca.crt", + data: "test-service-serving-ca-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/etcd-client-ca.key", + data: "test-etcdCA-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/etcd-client-ca.crt", + data: "test-etcdCA-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/cluster-apiserver-ca.key", + data: "test-cluster-apiserver-cert-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/cluster-apiserver-ca.crt", + data: "test-cluster-apiserver-cert-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/etcd-client.key", + data: "test-etcd-client-cert-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/etcd-client.crt", + data: "test-etcd-client-cert-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/apiserver.key", + data: "test-apiserver-cert-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/apiserver.crt", + data: "test-apiserver-cert-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/openshift-apiserver.key", + data: "test-openshift-apiserver-cert-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/openshift-apiserver.crt", + data: "test-openshift-apiserver-cert-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/apiserver-proxy.key", + data: "test-apiserver-proxy-cert-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/apiserver-proxy.crt", + data: "test-apiserver-proxy-cert-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/admin.key", + data: "test-admin-cert-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/admin.crt", + data: "test-admin-cert-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/kubelet.key", + data: "test-kubelet-cert-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/kubelet.crt", + data: "test-kubelet-cert-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/tnc.key", + data: "test-tnc-cert-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/tnc.crt", + data: "test-tnc-cert-pub", + }, + fileAssertion{ + path: "/opt/tectonic/tls/service-account.key", + data: "test-service-account-cert-priv", + }, + fileAssertion{ + path: "/opt/tectonic/tls/service-account.crt", + data: "test-service-account-cert-pub", + }, + ) + + assertSystemdUnitsInIgnitionConfig( + t, + bootstrapState.Contents[0].Data, + systemdUnitAssertion{ + name: "bootkube.service", + contents: templates.BootkubeSystemdContents, + }, + systemdUnitAssertion{ + name: "tectonic.service", + contents: templates.TectonicSystemdContents, + }, + systemdUnitAssertion{ + name: "kubelet.service", + contents: expectedKubeletService, + }, + ) + + assertUsersInIgnitionConfig( + t, + bootstrapState.Contents[0].Data, + userAssertion{ + name: "core", + sshKey: "test-admin-ssh-key", + }, + ) +} + +const ( + expectedBootkubeSh = `#!/usr/bin/env bash +set -e + +mkdir --parents /etc/kubernetes/manifests/ + +if [ ! -d kco-bootstrap ] +then + echo "Rendering Kubernetes core manifests..." + + # shellcheck disable=SC2154 + podman run \ + --volume "$PWD:/assets:z" \ + --volume /etc/kubernetes:/etc/kubernetes:z \ + "quay.io/coreos/kube-core-renderer-dev:436b1b4395ae54d866edc88864c9b01797cebac1" \ + --config=/assets/kco-config.yaml \ + --output=/assets/kco-bootstrap + + cp --recursive kco-bootstrap/bootstrap-configs /etc/kubernetes/bootstrap-configs + cp --recursive kco-bootstrap/bootstrap-manifests . + cp --recursive kco-bootstrap/manifests . +fi + +if [ ! -d mco-bootstrap ] +then + echo "Rendering MCO manifests..." + + # shellcheck disable=SC2154 + podman run \ + --user 0 \ + --volume "$PWD:/assets:z" \ + "docker.io/openshift/origin-machine-config-operator:v4.0.0" \ + bootstrap \ + --etcd-ca=/assets/tls/etcd-client-ca.crt \ + --root-ca=/assets/tls/root-ca.crt \ + --config-file=/assets/manifests/cluster-config.yaml \ + --dest-dir=/assets/mco-bootstrap \ + --images-json-configmap=/assets/manifests/machine-config-operator-01-images-configmap.yaml + + # Bootstrap MachineConfigController uses /etc/mcc/bootstrap/manifests/ dir to + # 1. read the controller config rendered by MachineConfigOperator + # 2. read the default MachineConfigPools rendered by MachineConfigOperator + # 3. read any additional MachineConfigs that are needed for the default MachineConfigPools. + mkdir --parents /etc/mcc/bootstrap/ + cp --recursive mco-bootstrap/manifests /etc/mcc/bootstrap/manifests + cp mco-bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/ + + # /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets. + mkdir --parents /etc/ssl/mcs/ + cp tls/machine-config-server.crt /etc/ssl/mcs/tls.crt + cp tls/machine-config-server.key /etc/ssl/mcs/tls.key +fi + +# We originally wanted to run the etcd cert signer as +# a static pod, but kubelet could't remove static pod +# when API server is not up, so we have to run this as +# podman container. +# See https://github.com/kubernetes/kubernetes/issues/43292 + +echo "Starting etcd certificate signer..." + +trap "podman rm --force etcd-signer" ERR + +# shellcheck disable=SC2154 +podman run \ + --name etcd-signer \ + --detach \ + --volume /opt/tectonic/tls:/opt/tectonic/tls:ro,z \ + --network host \ + "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6" \ + serve \ + --cacrt=/opt/tectonic/tls/etcd-client-ca.crt \ + --cakey=/opt/tectonic/tls/etcd-client-ca.key \ + --servcrt=/opt/tectonic/tls/apiserver.crt \ + --servkey=/opt/tectonic/tls/apiserver.key \ + --address=0.0.0.0:6443 \ + --csrdir=/tmp \ + --peercertdur=26280h \ + --servercertdur=26280h + +echo "Waiting for etcd cluster..." + +# Wait for the etcd cluster to come up. +set +e +# shellcheck disable=SC2154,SC2086 +until podman run \ + --rm \ + --network host \ + --name etcdctl \ + --env ETCDCTL_API=3 \ + --volume /opt/tectonic/tls:/opt/tectonic/tls:ro,z \ + "quay.io/coreos/etcd:v3.2.14" \ + /usr/local/bin/etcdctl \ + --dial-timeout=10m \ + --cacert=/opt/tectonic/tls/etcd-client-ca.crt \ + --cert=/opt/tectonic/tls/etcd-client.crt \ + --key=/opt/tectonic/tls/etcd-client.key \ + --endpoints=https://test-cluster-etcd-0.test-domain:2379,https://test-cluster-etcd-1.test-domain:2379,https://test-cluster-etcd-2.test-domain:2379 \ + endpoint health +do + echo "etcdctl failed. Retrying in 5 seconds..." + sleep 5 +done +set -e + +echo "etcd cluster up. Killing etcd certificate signer..." + +podman rm --force etcd-signer +rm --force /etc/kubernetes/manifests/machineconfigoperator-bootstrap-pod.yaml + +echo "Starting bootkube..." + +# shellcheck disable=SC2154 +podman run \ + --rm \ + --volume "$PWD:/assets:z" \ + --volume /etc/kubernetes:/etc/kubernetes:z \ + --network=host \ + --entrypoint=/bootkube \ + "quay.io/coreos/bootkube:v0.10.0" \ + start --asset-dir=/assets` + + expectedTectonicSh = `#!/usr/bin/env bash +set -e + +KUBECONFIG="$1" + +kubectl() { + echo "Executing kubectl $*" >&2 + while true + do + set +e + out=$(oc --config="$KUBECONFIG" "$@" 2>&1) + status=$? + set -e + + if grep --quiet "AlreadyExists" <<< "$out" + then + echo "$out, skipping" >&2 + return + fi + + echo "$out" + if [ "$status" -eq 0 ] + then + return + fi + + echo "kubectl $* failed. Retrying in 5 seconds..." >&2 + sleep 5 + done +} + +wait_for_pods() { + echo "Waiting for pods in namespace $1..." + while true + do + out=$(kubectl --namespace "$1" get pods --output custom-columns=STATUS:.status.phase,NAME:.metadata.name --no-headers=true) + echo "$out" + + # make sure kubectl returns at least one status + if [ "$(wc --lines <<< "$out")" -eq 0 ] + then + echo "No pods were found. Waiting for 5 seconds..." + sleep 5 + continue + fi + + if ! grep --invert-match '^Running' <<< "$out" + then + return + fi + + echo "Not all pods available yet. Waiting for 5 seconds..." + sleep 5 + done + set -e +} + +# Wait for Kubernetes pods +wait_for_pods kube-system + +echo "Creating initial roles..." +kubectl delete --filename rbac/role-admin.yaml + +kubectl create --filename ingress/svc-account.yaml +kubectl create --filename rbac/role-admin.yaml +kubectl create --filename rbac/role-user.yaml +kubectl create --filename rbac/binding-admin.yaml +kubectl create --filename rbac/binding-discovery.yaml + +echo "Creating cluster config for Tectonic..." +kubectl create --filename cluster-config.yaml +kubectl create --filename ingress/cluster-config.yaml + +echo "Creating Tectonic secrets..." +kubectl create --filename secrets/pull.json +kubectl create --filename secrets/ingress-tls.yaml +kubectl create --filename secrets/ca-cert.yaml +kubectl create --filename ingress/pull.json + +echo "Creating operators..." +kubectl create --filename security/priviledged-scc-tectonic.yaml +kubectl create --filename updater/tectonic-channel-operator-kind.yaml +kubectl create --filename updater/app-version-kind.yaml +kubectl create --filename updater/migration-status-kind.yaml + +kubectl --namespace=tectonic-system get customresourcedefinition channeloperatorconfigs.tco.coreos.com +kubectl create --filename updater/tectonic-channel-operator-config.yaml + +kubectl create --filename updater/operators/kube-core-operator.yaml +kubectl create --filename updater/operators/tectonic-channel-operator.yaml +kubectl create --filename updater/operators/kube-addon-operator.yaml +kubectl create --filename updater/operators/tectonic-alm-operator.yaml +kubectl create --filename updater/operators/tectonic-utility-operator.yaml +kubectl create --filename updater/operators/tectonic-ingress-controller-operator.yaml + +kubectl --namespace=tectonic-system get customresourcedefinition appversions.tco.coreos.com +kubectl create --filename updater/app_versions/app-version-tectonic-cluster.yaml +kubectl create --filename updater/app_versions/app-version-kube-core.yaml +kubectl create --filename updater/app_versions/app-version-kube-addon.yaml +kubectl create --filename updater/app_versions/app-version-tectonic-alm.yaml +kubectl create --filename updater/app_versions/app-version-tectonic-utility.yaml +kubectl create --filename updater/app_versions/app-version-tectonic-ingress.yaml + +# Wait for Tectonic pods +wait_for_pods tectonic-system + +echo "Tectonic installation is done"` + + // TODO (staebler): --cluster-dns should be 10.0.1.10 instead of 0:a::ffff:a00:100. + expectedKubeletService = `[Unit] +Description=Kubernetes Kubelet +Wants=rpc-statd.service + +[Service] +ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests +ExecStartPre=/usr/bin/bash -c "gawk '/certificate-authority-data/ {print $2}' /etc/kubernetes/kubeconfig | base64 --decode > /etc/kubernetes/ca.crt" + +ExecStart=/usr/bin/hyperkube \ + kubelet \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --rotate-certificates \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --cni-bin-dir=/var/lib/cni/bin \ + --network-plugin=cni \ + --lock-file=/var/run/lock/kubelet.lock \ + --exit-on-lock-contention \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged \ + --node-labels=node-role.kubernetes.io/bootstrap \ + --register-with-taints=node-role.kubernetes.io/bootstrap=:NoSchedule \ + --minimum-container-ttl-duration=6m0s \ + --cluster-dns=0:a::ffff:a00:100 \ + --cluster-domain=cluster.local \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cloud-provider=aws \ + --anonymous-auth=false \ + --cgroup-driver=systemd \ + \ + \ + +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target` +) diff --git a/pkg/asset/ignition/doc.go b/pkg/asset/ignition/doc.go new file mode 100644 index 00000000000..09c1cff1d2e --- /dev/null +++ b/pkg/asset/ignition/doc.go @@ -0,0 +1,2 @@ +// Package ignition generates the ignition config assets. +package ignition diff --git a/pkg/asset/ignition/master.go b/pkg/asset/ignition/master.go new file mode 100644 index 00000000000..345b108ddaf --- /dev/null +++ b/pkg/asset/ignition/master.go @@ -0,0 +1,58 @@ +package ignition + +import ( + "fmt" + "path/filepath" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/tls" +) + +// master is an asset that generates the ignition config for master nodes. +type master struct { + directory string + installConfig asset.Asset + rootCA asset.Asset +} + +var _ asset.Asset = (*master)(nil) + +// newMaster generates a new master asset. +func newMaster( + directory string, + installConfigStock installconfig.Stock, + tlsStock tls.Stock, +) *master { + return &master{ + directory: directory, + installConfig: installConfigStock.InstallConfig(), + rootCA: tlsStock.RootCA(), + } +} + +// Dependencies returns the assets on which the master asset depends. +func (a *master) Dependencies() []asset.Asset { + return []asset.Asset{ + a.installConfig, + a.rootCA, + } +} + +// Generate generates the ignition config for the master asset. +func (a *master) Generate(dependencies map[asset.Asset]*asset.State) (*asset.State, error) { + installConfig, err := installconfig.GetInstallConfig(a.installConfig, dependencies) + if err != nil { + return nil, err + } + + state := &asset.State{ + Contents: make([]asset.Content, masterCount(installConfig)), + } + for i := range state.Contents { + state.Contents[i].Name = filepath.Join(a.directory, fmt.Sprintf("master-%d.ign", i)) + state.Contents[i].Data = pointerIgnitionConfig(installConfig, dependencies[a.rootCA].Contents[0].Data, "master", fmt.Sprintf("etcd_index=%d", i)) + } + + return state, nil +} diff --git a/pkg/asset/ignition/master_test.go b/pkg/asset/ignition/master_test.go new file mode 100644 index 00000000000..6f8de1a8cd0 --- /dev/null +++ b/pkg/asset/ignition/master_test.go @@ -0,0 +1,49 @@ +package ignition + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/openshift/installer/pkg/asset" +) + +// TestMasterGenerate tests generating the master asset. +func TestMasterGenerate(t *testing.T) { + installConfig := ` +metadata: + name: test-cluster +baseDomain: test-domain +networking: + ServiceCIDR: 10.0.1.0/24 +platform: + aws: + region: us-east +machines: +- name: master + replicas: 3 +` + installConfigAsset := &testAsset{"install-config"} + rootCAAsset := &testAsset{"rootCA"} + master := &master{ + directory: "test-directory", + installConfig: installConfigAsset, + rootCA: rootCAAsset, + } + dependencies := map[asset.Asset]*asset.State{ + installConfigAsset: stateWithContentsData(installConfig), + rootCAAsset: stateWithContentsData("test-rootCA-priv", "test-rootCA-pub"), + } + masterState, err := master.Generate(dependencies) + assert.NoError(t, err, "unexpected error generating master asset") + expectedIgnitionConfigNames := []string{ + "test-directory/master-0.ign", + "test-directory/master-1.ign", + "test-directory/master-2.ign", + } + actualIgnitionConfigNames := make([]string, len(masterState.Contents)) + for i, c := range masterState.Contents { + actualIgnitionConfigNames[i] = c.Name + } + assert.Equal(t, expectedIgnitionConfigNames, actualIgnitionConfigNames, "unexpected names for master ignition configs") +} diff --git a/pkg/asset/ignition/node.go b/pkg/asset/ignition/node.go new file mode 100644 index 00000000000..f14611d0f43 --- /dev/null +++ b/pkg/asset/ignition/node.go @@ -0,0 +1,85 @@ +package ignition + +import ( + "encoding/json" + "fmt" + "net/url" + + ignition "github.com/coreos/ignition/config/v2_2/types" + "github.com/vincent-petithory/dataurl" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/types" +) + +const ( + // keyCertAssetKeyIndex is the index of the private key in a key-pair asset. + keyCertAssetKeyIndex = 0 + // keyCertAssetCrtIndex is the index of the public key in a key-pair asset. + keyCertAssetCrtIndex = 1 +) + +// fileFromAsset creates an ignition-config file with the contents from the +// specified index in the specified asset state. +func fileFromAsset(path string, mode int, assetState *asset.State, contentIndex int) ignition.File { + return fileFromBytes(path, mode, assetState.Contents[contentIndex].Data) +} + +// fileFromAsset creates an ignition-config file with the given contents. +func fileFromBytes(path string, mode int, contents []byte) ignition.File { + return ignition.File{ + Node: ignition.Node{ + Filesystem: "root", + Path: path, + }, + FileEmbedded1: ignition.FileEmbedded1{ + Mode: &mode, + Contents: ignition.FileContents{ + Source: dataurl.EncodeBytes(contents), + }, + }, + } +} + +// masterCount determines the number of master nodes from the install config, +// defaulting to one if it is unspecified. +func masterCount(installConfig *types.InstallConfig) int { + for _, m := range installConfig.Machines { + if m.Name == "master" && m.Replicas != nil { + return int(*m.Replicas) + } + } + return 1 +} + +// pointerIgnitionConfig generates a config which references the remote config +// served by the machine config server. +func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, role string, query string) []byte { + data, err := json.Marshal(ignition.Config{ + Ignition: ignition.Ignition{ + Config: ignition.IgnitionConfig{ + Append: []ignition.ConfigReference{{ + Source: func() *url.URL { + return &url.URL{ + Scheme: "https", + Host: fmt.Sprintf("%s-tnc.%s:49500", installConfig.Name, installConfig.BaseDomain), + Path: fmt.Sprintf("/config/%s", role), + RawQuery: query, + } + }().String(), + }}, + }, + Security: ignition.Security{ + TLS: ignition.TLS{ + CertificateAuthorities: []ignition.CaReference{{ + Source: dataurl.EncodeBytes(rootCA), + }}, + }, + }, + }, + }) + if err != nil { + panic(fmt.Sprintf("Failed to marshal pointer Ignition config: %v", err)) + } + return data +} diff --git a/pkg/asset/ignition/stock.go b/pkg/asset/ignition/stock.go new file mode 100644 index 00000000000..a5e81ec83ae --- /dev/null +++ b/pkg/asset/ignition/stock.go @@ -0,0 +1,49 @@ +package ignition + +import ( + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/kubeconfig" + "github.com/openshift/installer/pkg/asset/tls" +) + +// Stock is the stock of InstallConfig assets that can be generated. +type Stock interface { + // BootstrapIgnition is the asset that generates the bootstrap.ign ignition + // config file for the bootstrap node. + BootstrapIgnition() asset.Asset + // MasterIgnition is the asset that generates the master.ign ignition config + // files for the master nodes. + MasterIgnition() asset.Asset + // WorkerIgnition is the asset that generates the worker.ign ignition config + // file for the worker nodes. + WorkerIgnition() asset.Asset +} + +// StockImpl is the implementation of the ignition asset stock. +type StockImpl struct { + boostrap asset.Asset + master asset.Asset + worker asset.Asset +} + +// EstablishStock establishes the stock of assets in the specified directory. +func (s *StockImpl) EstablishStock( + directory string, + installConfigStock installconfig.Stock, + tlsStock tls.Stock, + kubeconfigStock kubeconfig.Stock, +) { + s.boostrap = newBootstrap(directory, installConfigStock, tlsStock, kubeconfigStock) + s.master = newMaster(directory, installConfigStock, tlsStock) + s.worker = newWorker(directory, installConfigStock, tlsStock) +} + +// BootstrapIgnition returns the bootstrap asset. +func (s *StockImpl) BootstrapIgnition() asset.Asset { return s.boostrap } + +// MasterIgnition returns the master asset. +func (s *StockImpl) MasterIgnition() asset.Asset { return s.master } + +// WorkerIgnition returns the worker asset. +func (s *StockImpl) WorkerIgnition() asset.Asset { return s.worker } diff --git a/pkg/asset/ignition/templates/BUILD.bazel b/pkg/asset/ignition/templates/BUILD.bazel new file mode 100644 index 00000000000..eec25a1dfdc --- /dev/null +++ b/pkg/asset/ignition/templates/BUILD.bazel @@ -0,0 +1,13 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "bootkube.go", + "doc.go", + "kubelet.go", + "tectonic.go", + ], + importpath = "github.com/openshift/installer/pkg/asset/ignition/templates", + visibility = ["//visibility:public"], +) diff --git a/pkg/asset/ignition/templates/bootkube.go b/pkg/asset/ignition/templates/bootkube.go new file mode 100644 index 00000000000..2307fd44501 --- /dev/null +++ b/pkg/asset/ignition/templates/bootkube.go @@ -0,0 +1,141 @@ +package templates + +const ( + // BootkubeSystemdContents is a service for running bootkube on the bootstrap + // nodes + BootkubeSystemdContents = `[Unit] +Description=Bootstrap a Kubernetes cluster +Wants=kubelet.service +After=kubelet.service + +[Service] +WorkingDirectory=/opt/tectonic + +ExecStart=/opt/tectonic/bootkube.sh + +Restart=on-failure +RestartSec=5s` + + // BootkubeShFileContents is a script file for running bootkube on the + // bootstrap nodes. + BootkubeShFileContents = `#!/usr/bin/env bash +set -e + +mkdir --parents /etc/kubernetes/manifests/ + +if [ ! -d kco-bootstrap ] +then + echo "Rendering Kubernetes core manifests..." + + # shellcheck disable=SC2154 + podman run \ + --volume "$PWD:/assets:z" \ + --volume /etc/kubernetes:/etc/kubernetes:z \ + "{{.KubeCoreRenderImage}}" \ + --config=/assets/kco-config.yaml \ + --output=/assets/kco-bootstrap + + cp --recursive kco-bootstrap/bootstrap-configs /etc/kubernetes/bootstrap-configs + cp --recursive kco-bootstrap/bootstrap-manifests . + cp --recursive kco-bootstrap/manifests . +fi + +if [ ! -d mco-bootstrap ] +then + echo "Rendering MCO manifests..." + + # shellcheck disable=SC2154 + podman run \ + --user 0 \ + --volume "$PWD:/assets:z" \ + "{{.MachineConfigOperatorImage}}" \ + bootstrap \ + --etcd-ca=/assets/tls/etcd-client-ca.crt \ + --root-ca=/assets/tls/root-ca.crt \ + --config-file=/assets/manifests/cluster-config.yaml \ + --dest-dir=/assets/mco-bootstrap \ + --images-json-configmap=/assets/manifests/machine-config-operator-01-images-configmap.yaml + + # Bootstrap MachineConfigController uses /etc/mcc/bootstrap/manifests/ dir to + # 1. read the controller config rendered by MachineConfigOperator + # 2. read the default MachineConfigPools rendered by MachineConfigOperator + # 3. read any additional MachineConfigs that are needed for the default MachineConfigPools. + mkdir --parents /etc/mcc/bootstrap/ + cp --recursive mco-bootstrap/manifests /etc/mcc/bootstrap/manifests + cp mco-bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/ + + # /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets. + mkdir --parents /etc/ssl/mcs/ + cp tls/machine-config-server.crt /etc/ssl/mcs/tls.crt + cp tls/machine-config-server.key /etc/ssl/mcs/tls.key +fi + +# We originally wanted to run the etcd cert signer as +# a static pod, but kubelet could't remove static pod +# when API server is not up, so we have to run this as +# podman container. +# See https://github.com/kubernetes/kubernetes/issues/43292 + +echo "Starting etcd certificate signer..." + +trap "podman rm --force etcd-signer" ERR + +# shellcheck disable=SC2154 +podman run \ + --name etcd-signer \ + --detach \ + --volume /opt/tectonic/tls:/opt/tectonic/tls:ro,z \ + --network host \ + "{{.EtcdCertSignerImage}}" \ + serve \ + --cacrt=/opt/tectonic/tls/etcd-client-ca.crt \ + --cakey=/opt/tectonic/tls/etcd-client-ca.key \ + --servcrt=/opt/tectonic/tls/apiserver.crt \ + --servkey=/opt/tectonic/tls/apiserver.key \ + --address=0.0.0.0:6443 \ + --csrdir=/tmp \ + --peercertdur=26280h \ + --servercertdur=26280h + +echo "Waiting for etcd cluster..." + +# Wait for the etcd cluster to come up. +set +e +# shellcheck disable=SC2154,SC2086 +until podman run \ + --rm \ + --network host \ + --name etcdctl \ + --env ETCDCTL_API=3 \ + --volume /opt/tectonic/tls:/opt/tectonic/tls:ro,z \ + "{{.EtcdctlImage}}" \ + /usr/local/bin/etcdctl \ + --dial-timeout=10m \ + --cacert=/opt/tectonic/tls/etcd-client-ca.crt \ + --cert=/opt/tectonic/tls/etcd-client.crt \ + --key=/opt/tectonic/tls/etcd-client.key \ + --endpoints={{.EtcdCluster}} \ + endpoint health +do + echo "etcdctl failed. Retrying in 5 seconds..." + sleep 5 +done +set -e + +echo "etcd cluster up. Killing etcd certificate signer..." + +podman rm --force etcd-signer +rm --force /etc/kubernetes/manifests/machineconfigoperator-bootstrap-pod.yaml + +echo "Starting bootkube..." + +# shellcheck disable=SC2154 +podman run \ + --rm \ + --volume "$PWD:/assets:z" \ + --volume /etc/kubernetes:/etc/kubernetes:z \ + --network=host \ + --entrypoint=/bootkube \ + "{{.BootkubeImage}}" \ + start --asset-dir=/assets` +) diff --git a/pkg/asset/ignition/templates/doc.go b/pkg/asset/ignition/templates/doc.go new file mode 100644 index 00000000000..9df1cc1a733 --- /dev/null +++ b/pkg/asset/ignition/templates/doc.go @@ -0,0 +1,3 @@ +// Package templates contains consts for the contents of files and systemd units +// to add to ignition configs. +package templates diff --git a/pkg/asset/ignition/templates/kubelet.go b/pkg/asset/ignition/templates/kubelet.go new file mode 100644 index 00000000000..6f9d72d784f --- /dev/null +++ b/pkg/asset/ignition/templates/kubelet.go @@ -0,0 +1,43 @@ +package templates + +const ( + // KubeletSystemdContents is a service for running the kubelet on the + // bootstrap nodes. + KubeletSystemdContents = `[Unit] +Description=Kubernetes Kubelet +Wants=rpc-statd.service + +[Service] +ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests +ExecStartPre=/usr/bin/bash -c "gawk '/certificate-authority-data/ {print $2}' /etc/kubernetes/kubeconfig | base64 --decode > /etc/kubernetes/ca.crt" + +ExecStart=/usr/bin/hyperkube \ + kubelet \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --rotate-certificates \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --cni-bin-dir=/var/lib/cni/bin \ + --network-plugin=cni \ + --lock-file=/var/run/lock/kubelet.lock \ + --exit-on-lock-contention \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged \ + --node-labels=node-role.kubernetes.io/bootstrap \ + --register-with-taints=node-role.kubernetes.io/bootstrap=:NoSchedule \ + --minimum-container-ttl-duration=6m0s \ + --cluster-dns={{.ClusterDNSIP}} \ + --cluster-domain=cluster.local \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cloud-provider={{.CloudProvider}} \ + --anonymous-auth=false \ + --cgroup-driver=systemd \ + {{.CloudProviderConfig}} \ + {{.DebugConfig}} \ + +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target` +) diff --git a/pkg/asset/ignition/templates/tectonic.go b/pkg/asset/ignition/templates/tectonic.go new file mode 100644 index 00000000000..9c0b86fc867 --- /dev/null +++ b/pkg/asset/ignition/templates/tectonic.go @@ -0,0 +1,130 @@ +package templates + +const ( + // TectonicSystemdContents is a service that runs tectonic on the masters. + TectonicSystemdContents = `[Unit] +Description=Bootstrap a Tectonic cluster +Wants=bootkube.service +After=bootkube.service + +[Service] +WorkingDirectory=/opt/tectonic/tectonic + +ExecStart=/opt/tectonic/tectonic.sh /opt/tectonic/auth/kubeconfig + +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target` + + // TectonicShFileContents is a script file for running tectonic on bootstrap + // nodes. + TectonicShFileContents = `#!/usr/bin/env bash +set -e + +KUBECONFIG="$1" + +kubectl() { + echo "Executing kubectl $*" >&2 + while true + do + set +e + out=$(oc --config="$KUBECONFIG" "$@" 2>&1) + status=$? + set -e + + if grep --quiet "AlreadyExists" <<< "$out" + then + echo "$out, skipping" >&2 + return + fi + + echo "$out" + if [ "$status" -eq 0 ] + then + return + fi + + echo "kubectl $* failed. Retrying in 5 seconds..." >&2 + sleep 5 + done +} + +wait_for_pods() { + echo "Waiting for pods in namespace $1..." + while true + do + out=$(kubectl --namespace "$1" get pods --output custom-columns=STATUS:.status.phase,NAME:.metadata.name --no-headers=true) + echo "$out" + + # make sure kubectl returns at least one status + if [ "$(wc --lines <<< "$out")" -eq 0 ] + then + echo "No pods were found. Waiting for 5 seconds..." + sleep 5 + continue + fi + + if ! grep --invert-match '^Running' <<< "$out" + then + return + fi + + echo "Not all pods available yet. Waiting for 5 seconds..." + sleep 5 + done + set -e +} + +# Wait for Kubernetes pods +wait_for_pods kube-system + +echo "Creating initial roles..." +kubectl delete --filename rbac/role-admin.yaml + +kubectl create --filename ingress/svc-account.yaml +kubectl create --filename rbac/role-admin.yaml +kubectl create --filename rbac/role-user.yaml +kubectl create --filename rbac/binding-admin.yaml +kubectl create --filename rbac/binding-discovery.yaml + +echo "Creating cluster config for Tectonic..." +kubectl create --filename cluster-config.yaml +kubectl create --filename ingress/cluster-config.yaml + +echo "Creating Tectonic secrets..." +kubectl create --filename secrets/pull.json +kubectl create --filename secrets/ingress-tls.yaml +kubectl create --filename secrets/ca-cert.yaml +kubectl create --filename ingress/pull.json + +echo "Creating operators..." +kubectl create --filename security/priviledged-scc-tectonic.yaml +kubectl create --filename updater/tectonic-channel-operator-kind.yaml +kubectl create --filename updater/app-version-kind.yaml +kubectl create --filename updater/migration-status-kind.yaml + +kubectl --namespace=tectonic-system get customresourcedefinition channeloperatorconfigs.tco.coreos.com +kubectl create --filename updater/tectonic-channel-operator-config.yaml + +kubectl create --filename updater/operators/kube-core-operator.yaml +kubectl create --filename updater/operators/tectonic-channel-operator.yaml +kubectl create --filename updater/operators/kube-addon-operator.yaml +kubectl create --filename updater/operators/tectonic-alm-operator.yaml +kubectl create --filename updater/operators/tectonic-utility-operator.yaml +kubectl create --filename updater/operators/tectonic-ingress-controller-operator.yaml + +kubectl --namespace=tectonic-system get customresourcedefinition appversions.tco.coreos.com +kubectl create --filename updater/app_versions/app-version-tectonic-cluster.yaml +kubectl create --filename updater/app_versions/app-version-kube-core.yaml +kubectl create --filename updater/app_versions/app-version-kube-addon.yaml +kubectl create --filename updater/app_versions/app-version-tectonic-alm.yaml +kubectl create --filename updater/app_versions/app-version-tectonic-utility.yaml +kubectl create --filename updater/app_versions/app-version-tectonic-ingress.yaml + +# Wait for Tectonic pods +wait_for_pods tectonic-system + +echo "Tectonic installation is done"` +) diff --git a/pkg/asset/ignition/testasset_test.go b/pkg/asset/ignition/testasset_test.go new file mode 100644 index 00000000000..27d8adb6e8c --- /dev/null +++ b/pkg/asset/ignition/testasset_test.go @@ -0,0 +1,27 @@ +package ignition + +import ( + "github.com/openshift/installer/pkg/asset" +) + +type testAsset struct { + name string +} + +func (a *testAsset) Dependencies() []asset.Asset { + return []asset.Asset{} +} + +func (a *testAsset) Generate(map[asset.Asset]*asset.State) (*asset.State, error) { + return nil, nil +} + +func stateWithContentsData(contentsData ...string) *asset.State { + state := &asset.State{ + Contents: make([]asset.Content, len(contentsData)), + } + for i, d := range contentsData { + state.Contents[i].Data = []byte(d) + } + return state +} diff --git a/pkg/asset/ignition/testutils_test.go b/pkg/asset/ignition/testutils_test.go new file mode 100644 index 00000000000..3ed0e13ced3 --- /dev/null +++ b/pkg/asset/ignition/testutils_test.go @@ -0,0 +1,250 @@ +package ignition + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vincent-petithory/dataurl" +) + +type fileAssertion struct { + path string + data string + additional func(*testing.T, map[string]interface{}) bool +} + +// assertFilesInIgnitionConfig asserts that the specified ignition config +// contains exactly the files enumerated in fileAssertions. +func assertFilesInIgnitionConfig( + t *testing.T, + ignitionConfig []byte, + fileAssertions ...fileAssertion, +) bool { + var ic map[string]interface{} + if err := json.Unmarshal(ignitionConfig, &ic); err != nil { + return assert.NoError(t, err, "unexpected error unmarshaling ignition config") + } + storage, ok := ic["storage"] + if !assert.True(t, ok, "No storage in ignition config") { + return false + } + files, ok := storage.(map[string]interface{})["files"] + if !assert.True(t, ok, "No files in ignition config") { + return false + } + expectedFilePaths := make([]string, len(fileAssertions)) + for i, a := range fileAssertions { + expectedFilePaths[i] = a.path + } + filesList := files.([]interface{}) + actualFilePaths := make([]string, len(filesList)) + for i, f := range filesList { + path, ok := f.(map[string]interface{})["path"] + if !assert.True(t, ok, "file has no path: %+v", f) { + return false + } + actualFilePaths[i] = path.(string) + } + if !assert.Equal(t, expectedFilePaths, actualFilePaths, "Unexpected file paths") { + return false + } + for _, f := range filesList { + file := f.(map[string]interface{}) + path := file["path"] + var fa fileAssertion + for _, a := range fileAssertions { + if a.path != path { + continue + } + fa = a + } + contents, ok := file["contents"] + if !assert.True(t, ok, "file %q has no contents", path) { + return false + } + source, ok := contents.(map[string]interface{})["source"] + if !assert.True(t, ok, "file %q has no source", path) { + return false + } + url, err := dataurl.DecodeString(source.(string)) + if !assert.NoError(t, err, "unexpected error decoding dataurl in file %q", path) { + return false + } + if !assert.Equal(t, fa.data, string(url.Data), "unexpected data in file %q", path) { + return false + } + if fa.additional != nil { + if !fa.additional(t, file) { + return false + } + } + } + return true +} + +type systemdUnitAssertion struct { + name string + dropinName string + contents string + additional func(*testing.T, map[string]interface{}) bool +} + +// assertSystemdUnitsInIgnitionConfig asserts that the specified ignition config +// contains exactly the systemd units enumerated in unitAssertions. +func assertSystemdUnitsInIgnitionConfig( + t *testing.T, + ignitionConfig []byte, + unitAssertions ...systemdUnitAssertion, +) bool { + var ic map[string]interface{} + if err := json.Unmarshal(ignitionConfig, &ic); err != nil { + return assert.NoError(t, err, "unexpected error unmarshaling ignition config") + } + systemd, ok := ic["systemd"] + if !assert.True(t, ok, "No systemd in ignition config") { + return false + } + units, ok := systemd.(map[string]interface{})["units"] + if !assert.True(t, ok, "No units in ignition config") { + return false + } + expectedUnitNames := make([]string, len(unitAssertions)) + for i, a := range unitAssertions { + expectedUnitNames[i] = a.name + } + unitsList := units.([]interface{}) + actualUnitNames := make([]string, len(unitsList)) + for i, u := range unitsList { + name, ok := u.(map[string]interface{})["name"] + if !assert.True(t, ok, "unit has no name: %+v", u) { + return false + } + actualUnitNames[i] = name.(string) + } + if !assert.Equal(t, expectedUnitNames, actualUnitNames, "Unexpected unit names") { + return false + } + for _, u := range unitsList { + unit := u.(map[string]interface{}) + name := unit["name"] + var ua systemdUnitAssertion + for _, a := range unitAssertions { + if a.name != name { + continue + } + ua = a + } + contentsParent := unit + if ua.dropinName != "" { + dropins, ok := unit["dropins"] + if !assert.True(t, ok, "no dropins in systemd unit %q", name) { + return false + } + dropinsList := dropins.([]interface{}) + if !assert.Equal(t, 1, len(dropinsList), "unexpected number of dropins in systemd unit %q", name) { + return false + } + dropin := dropinsList[0].(map[string]interface{}) + dropinName, ok := dropin["name"] + if !assert.True(t, ok, "no name in dropin in systemd unit %q", name) { + return false + } + if !assert.Equal(t, ua.dropinName, dropinName.(string), "unexpected dropin name in systemd unit %q", name) { + return false + } + contentsParent = dropin + } + contents, contentsOK := contentsParent["contents"] + if ua.contents != "" { + if !assert.True(t, contentsOK, "no contents in systemd unit %q", name) { + return false + } + if !assert.Equal(t, ua.contents, contents.(string), "unexpected contents in systemd unit %q", name) { + return false + } + } else { + if !assert.False(t, contentsOK, "unexpected contents in systemd unit %q", name) { + return false + } + } + if ua.additional != nil { + if !ua.additional(t, unit) { + return false + } + } + } + return true +} + +type userAssertion struct { + name string + sshKey string + additional func(*testing.T, map[string]interface{}) bool +} + +// assertUsersInIgnitionConfig asserts that the specified ignition config +// contains exactly the users enumerated in userAssertions. +func assertUsersInIgnitionConfig( + t *testing.T, + ignitionConfig []byte, + userAssertions ...userAssertion, +) bool { + var ic map[string]interface{} + if err := json.Unmarshal(ignitionConfig, &ic); err != nil { + return assert.NoError(t, err, "unexpected error unmarshaling ignition config") + } + passwd, ok := ic["passwd"] + if !assert.True(t, ok, "No passwd in ignition config") { + return false + } + users, ok := passwd.(map[string]interface{})["users"] + if !assert.True(t, ok, "No users in ignition config") { + return false + } + expectedUserNames := make([]string, len(userAssertions)) + for i, a := range userAssertions { + expectedUserNames[i] = a.name + } + usersList := users.([]interface{}) + actualUserNames := make([]string, len(usersList)) + for i, u := range usersList { + name, ok := u.(map[string]interface{})["name"] + if !assert.True(t, ok, "user has no name: %+v", u) { + return false + } + actualUserNames[i] = name.(string) + } + if !assert.Equal(t, expectedUserNames, actualUserNames, "Unexpected user names") { + return false + } + for _, u := range usersList { + user := u.(map[string]interface{}) + name := user["name"] + var ua userAssertion + for _, a := range userAssertions { + if a.name != name { + continue + } + ua = a + } + sshAuthorizedKeys, ok := user["sshAuthorizedKeys"] + if !assert.True(t, ok, "no sshAuthorizedKeys in user %q", name) { + return false + } + sshAuthorizedKeysList := sshAuthorizedKeys.([]interface{}) + if !assert.Equal(t, 1, len(sshAuthorizedKeysList), "unexpected number of sshAuthorizedKeys in user %q", name) { + return false + } + sshAuthorizedKey := sshAuthorizedKeysList[0].(string) + if !assert.Equal(t, ua.sshKey, sshAuthorizedKey, "unexpected ssh key in user %q", name) { + return false + } + if ua.additional != nil { + if !ua.additional(t, user) { + return false + } + } + } + return true +} diff --git a/pkg/asset/ignition/worker.go b/pkg/asset/ignition/worker.go new file mode 100644 index 00000000000..bb922cda23a --- /dev/null +++ b/pkg/asset/ignition/worker.go @@ -0,0 +1,54 @@ +package ignition + +import ( + "path/filepath" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/tls" +) + +// worker is an asset that generates the ignition config for worker nodes. +type worker struct { + directory string + installConfig asset.Asset + rootCA asset.Asset +} + +var _ asset.Asset = (*worker)(nil) + +// newWorker creates a new worker asset. +func newWorker( + directory string, + installConfigStock installconfig.Stock, + tlsStock tls.Stock, +) *worker { + return &worker{ + directory: directory, + installConfig: installConfigStock.InstallConfig(), + rootCA: tlsStock.RootCA(), + } +} + +// Dependencies returns the assets on which the worker asset depends. +func (a *worker) Dependencies() []asset.Asset { + return []asset.Asset{ + a.installConfig, + a.rootCA, + } +} + +// Generate generates the ignition config for the worker asset. +func (a *worker) Generate(dependencies map[asset.Asset]*asset.State) (*asset.State, error) { + installConfig, err := installconfig.GetInstallConfig(a.installConfig, dependencies) + if err != nil { + return nil, err + } + + return &asset.State{ + Contents: []asset.Content{{ + Name: filepath.Join(a.directory, "worker.ign"), + Data: pointerIgnitionConfig(installConfig, dependencies[a.rootCA].Contents[0].Data, "worker", ""), + }}, + }, nil +} diff --git a/pkg/asset/ignition/worker_test.go b/pkg/asset/ignition/worker_test.go new file mode 100644 index 00000000000..0b8ffb23793 --- /dev/null +++ b/pkg/asset/ignition/worker_test.go @@ -0,0 +1,35 @@ +package ignition + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/openshift/installer/pkg/asset" +) + +// TestWorkerGenerate tests generating the worker asset. +func TestWorkerGenerate(t *testing.T) { + installConfig := ` +networking: + ServiceCIDR: 10.0.1.0/24 +platform: + aws: +region: us-east +` + installConfigAsset := &testAsset{"install-config"} + rootCAAsset := &testAsset{"rootCA"} + worker := &worker{ + directory: "test-directory", + installConfig: installConfigAsset, + rootCA: rootCAAsset, + } + dependencies := map[asset.Asset]*asset.State{ + installConfigAsset: stateWithContentsData(installConfig), + rootCAAsset: stateWithContentsData("test-rootCA-priv", "test-rootCA-pub"), + } + workerState, err := worker.Generate(dependencies) + assert.NoError(t, err, "unexpected error generating worker asset") + assert.Equal(t, 1, len(workerState.Contents), "unexpected number of contents in worker state") + assert.Equal(t, "test-directory/worker.ign", workerState.Contents[0].Name, "unexpected name for worker ignition config") +} diff --git a/pkg/asset/installconfig/BUILD.bazel b/pkg/asset/installconfig/BUILD.bazel index d603db4b0ad..f215e1ce4bf 100644 --- a/pkg/asset/installconfig/BUILD.bazel +++ b/pkg/asset/installconfig/BUILD.bazel @@ -15,7 +15,9 @@ go_library( deps = [ "//installer/pkg/validate:go_default_library", "//pkg/asset:go_default_library", + "//pkg/ipnet:go_default_library", "//pkg/types:go_default_library", + "//vendor/github.com/apparentlymart/go-cidr/cidr:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -31,6 +33,8 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/asset:go_default_library", + "//pkg/ipnet:go_default_library", + "//pkg/types:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", ], ) diff --git a/pkg/asset/installconfig/installconfig.go b/pkg/asset/installconfig/installconfig.go index fae7ade008e..858a7abecba 100644 --- a/pkg/asset/installconfig/installconfig.go +++ b/pkg/asset/installconfig/installconfig.go @@ -2,16 +2,24 @@ package installconfig import ( "fmt" + "net" "path/filepath" + "github.com/apparentlymart/go-cidr/cidr" "github.com/ghodss/yaml" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/ipnet" "github.com/openshift/installer/pkg/types" ) +var ( + defaultServiceCIDR = parseCIDR("10.3.0.0/16") + defaultPodCIDR = parseCIDR("10.2.0.0/16") +) + // installConfig generates the install-config.yml file. type installConfig struct { assetStock Stock @@ -56,7 +64,25 @@ func (a *installConfig) Generate(dependencies map[asset.Asset]*asset.State) (*as SSHKey: sshKey, }, BaseDomain: baseDomain, + Networking: types.Networking{ + ServiceCIDR: ipnet.IPNet{ + IPNet: defaultServiceCIDR, + }, + PodCIDR: ipnet.IPNet{ + IPNet: defaultPodCIDR, + }, + }, PullSecret: pullSecret, + Machines: []types.MachinePool{ + { + Name: "master", + Replicas: func(x int64) *int64 { return &x }(3), + }, + { + Name: "worker", + Replicas: func(x int64) *int64 { return &x }(3), + }, + }, } platformState := dependencies[a.assetStock.Platform()] @@ -106,3 +132,19 @@ func GetInstallConfig(installConfig asset.Asset, parents map[asset.Asset]*asset. return &cfg, nil } + +// ClusterDNSIP returns the string representation of the DNS server's IP +// address. +func ClusterDNSIP(installConfig *types.InstallConfig) (string, error) { + ip, err := cidr.Host(&installConfig.ServiceCIDR.IPNet, 10) + if err != nil { + return "", err + } + + return ip.String(), nil +} + +func parseCIDR(s string) net.IPNet { + _, cidr, _ := net.ParseCIDR(s) + return *cidr +} diff --git a/pkg/asset/installconfig/installconfig_test.go b/pkg/asset/installconfig/installconfig_test.go index 7e71cc9c730..e45607f410a 100644 --- a/pkg/asset/installconfig/installconfig_test.go +++ b/pkg/asset/installconfig/installconfig_test.go @@ -3,6 +3,7 @@ package installconfig import ( "fmt" "io/ioutil" + "net" "os" "path/filepath" "testing" @@ -10,6 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/ipnet" + "github.com/openshift/installer/pkg/types" ) type testAsset struct { @@ -158,13 +161,19 @@ func TestInstallConfigGenerate(t *testing.T) { sshKey: test-sshkey baseDomain: test-domain clusterID: test-cluster-id -machines: null +machines: +- name: master + platform: {} + replicas: 3 +- name: worker + platform: {} + replicas: 3 metadata: creationTimestamp: null name: test-cluster-name networking: - podCIDR: null - serviceCIDR: null + podCIDR: 10.2.0.0/16 + serviceCIDR: 10.3.0.0/16 type: "" platform: %s @@ -175,3 +184,20 @@ pullSecret: test-pull-secret }) } } + +// TestClusterDNSIP tests the ClusterDNSIP function. +func TestClusterDNSIP(t *testing.T) { + _, cidr, err := net.ParseCIDR("10.0.1.0/24") + assert.NoError(t, err, "unexpected error parsing CIDR") + installConfig := &types.InstallConfig{ + Networking: types.Networking{ + ServiceCIDR: ipnet.IPNet{ + IPNet: *cidr, + }, + }, + } + expected := "10.0.1.10" + actual, err := ClusterDNSIP(installConfig) + assert.NoError(t, err, "unexpected error get cluster DNS IP") + assert.Equal(t, expected, actual, "unexpected DNS IP") +} diff --git a/pkg/asset/stock/BUILD.bazel b/pkg/asset/stock/BUILD.bazel index 2a778c2afdc..00dd40529ff 100644 --- a/pkg/asset/stock/BUILD.bazel +++ b/pkg/asset/stock/BUILD.bazel @@ -9,6 +9,7 @@ go_library( importpath = "github.com/openshift/installer/pkg/asset/stock", visibility = ["//visibility:public"], deps = [ + "//pkg/asset/ignition:go_default_library", "//pkg/asset/installconfig:go_default_library", "//pkg/asset/kubeconfig:go_default_library", "//pkg/asset/tls:go_default_library", diff --git a/pkg/asset/stock/stock.go b/pkg/asset/stock/stock.go index 24b165457c4..9b5c49434ba 100644 --- a/pkg/asset/stock/stock.go +++ b/pkg/asset/stock/stock.go @@ -4,6 +4,7 @@ import ( "bufio" "os" + "github.com/openshift/installer/pkg/asset/ignition" "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/kubeconfig" "github.com/openshift/installer/pkg/asset/tls" @@ -14,6 +15,7 @@ type Stock struct { installConfigStock kubeconfigStock tlsStock + ignitionStock } type installConfigStock struct { @@ -28,6 +30,10 @@ type kubeconfigStock struct { kubeconfig.StockImpl } +type ignitionStock struct { + ignition.StockImpl +} + var _ installconfig.Stock = (*Stock)(nil) // EstablishStock establishes the stock of assets in the specified directory. @@ -37,6 +43,7 @@ func EstablishStock(directory string) *Stock { s.installConfigStock.EstablishStock(directory, inputReader) s.tlsStock.EstablishStock(directory, &s.installConfigStock) s.kubeconfigStock.EstablishStock(directory, &s.installConfigStock, &s.tlsStock) + s.ignitionStock.EstablishStock(directory, s, s, s) return s } diff --git a/tests/smoke/BUILD.bazel b/tests/smoke/BUILD.bazel index 721adeaa51e..428e9080886 100644 --- a/tests/smoke/BUILD.bazel +++ b/tests/smoke/BUILD.bazel @@ -2,12 +2,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test") go_test( name = "go_default_test", - pure = "on", srcs = [ "cluster_test.go", "common_test.go", "smoke_test.go", ], + pure = "on", visibility = ["//visibility:public"], deps = [ "//tests/smoke/vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",