diff --git a/cmd/cluster-etcd-operator/main.go b/cmd/cluster-etcd-operator/main.go index 59d893aa5d..f7af79081e 100644 --- a/cmd/cluster-etcd-operator/main.go +++ b/cmd/cluster-etcd-operator/main.go @@ -8,6 +8,7 @@ import ( "os" "time" + "github.com/openshift/cluster-etcd-operator/pkg/cmd/aio" operatorcmd "github.com/openshift/cluster-etcd-operator/pkg/cmd/operator" "github.com/openshift/cluster-etcd-operator/pkg/cmd/render" "github.com/openshift/cluster-etcd-operator/pkg/cmd/waitforceo" @@ -60,6 +61,7 @@ func NewSSCSCommand() *cobra.Command { cmd.AddCommand(prune.NewPrune()) cmd.AddCommand(certsyncpod.NewCertSyncControllerCommand(operator.CertConfigMaps, operator.CertSecrets)) cmd.AddCommand(waitforceo.NewWaitForCeoCommand(os.Stderr)) + cmd.AddCommand(aio.NewAIOCommand(os.Stderr)) return cmd } diff --git a/hack/aio-render.sh b/hack/aio-render.sh new file mode 100755 index 0000000000..dc5e0dc484 --- /dev/null +++ b/hack/aio-render.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# openshift-install --dir test/ create aio-config + +INSTALLER_ASSETS_DIR="$1" +IGNITION_CONFIG="${INSTALLER_ASSETS_DIR}/aio.ign" + +# oc adm release info registry.svc.ci.openshift.org/ocp/release:4.6.0-0.ci-2020-07-21-114552 -o json > release:4.6.0-0.ci-2020-07-21-114552-info.json +RELEASE_INFO="release:4.6.0-0.ci-2020-07-21-114552-info.json" + +mkdir -p ./assets/tls + +# Unpack the TLS assets from the ignition file +jq -c '.storage.files[] | {p:.path,c:.contents.source}' "${IGNITION_CONFIG}" | while read f t; do + p=$(echo $f | jq -r .p) + c=$(echo $f | jq -r .c) + + [[ "$p" != /opt/openshift/tls/* ]] && continue + + echo "${c#data:text/plain;charset=utf-8;base64,}" | base64 -d > "./assets/tls/$(basename $p)" +done + +image_for() { + jq -r '.references.spec.tags[] | select(.name =="tools") | .from.name' "${RELEASE_INFO}" +} + +MACHINE_CONFIG_ETCD_IMAGE=$(image_for etcd) + +./cluster-etcd-operator aio \ + --etcd-ca-cert=./assets/tls/etcd-signer.crt \ + --etcd-ca-key=./assets/tls/etcd-signer.key \ + --etcd-metric-ca-cert=./assets/tls/etcd-metric-signer.crt \ + --etcd-metric-ca-key=./assets/tls/etcd-metric-signer.key \ + --asset-output-dir=./assets/etcd-aio \ + --manifest-etcd-image="${MACHINE_CONFIG_ETCD_IMAGE}" diff --git a/hack/render.sh b/hack/render.sh new file mode 100755 index 0000000000..a62f4fd521 --- /dev/null +++ b/hack/render.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# openshift-install --dir test/ create manifests +# openshift-install --dir test/ create aio-config + +INSTALLER_ASSETS_DIR="$1" +IGNITION_CONFIG="${INSTALLER_ASSETS_DIR}/aio.ign" + +CLUSTER_DOMAIN="markmc.devcluster.openshift.com" + +# oc adm release info registry.svc.ci.openshift.org/ocp/release:4.6.0-0.ci-2020-07-21-114552 -o json > release:4.6.0-0.ci-2020-07-21-114552-info.json +RELEASE_INFO="release:4.6.0-0.ci-2020-07-21-114552-info.json" + +mkdir -p ./assets/tls + +# Unpack the TLS assets from the ignition file +jq -c '.storage.files[] | {p:.path,c:.contents.source}' "${IGNITION_CONFIG}" | while read f t; do + p=$(echo $f | jq -r .p) + c=$(echo $f | jq -r .c) + + [[ "$p" != /opt/openshift/tls/* ]] && continue + + echo "${c#data:text/plain;charset=utf-8;base64,}" | base64 -d > "./assets/tls/$(basename $p)" +done + +image_for() { + jq -r '.references.spec.tags[] | select(.name =="tools") | .from.name' "${RELEASE_INFO}" +} + +MACHINE_CONFIG_ETCD_IMAGE=$(image_for etcd) +CLUSTER_ETCD_OPERATOR_IMAGE=$(image_for cluster-etcd-operator) +MACHINE_CONFIG_OPERATOR_IMAGE=$(image_for machine-config-operator) +MACHINE_CONFIG_KUBE_CLIENT_AGENT_IMAGE=$(image_for kube-client-agent) + +./cluster-etcd-operator render \ + --templates-input-dir=./bindata/bootkube \ + --etcd-ca=./assets/tls/etcd-ca-bundle.crt \ + --etcd-metric-ca=./assets/tls/etcd-metric-ca-bundle.crt \ + --manifest-etcd-image="${MACHINE_CONFIG_ETCD_IMAGE}" \ + --etcd-discovery-domain="${CLUSTER_DOMAIN}" \ + --manifest-cluster-etcd-operator-image="${CLUSTER_ETCD_OPERATOR_IMAGE}" \ + --manifest-setup-etcd-env-image="${MACHINE_CONFIG_OPERATOR_IMAGE}" \ + --manifest-kube-client-agent-image="${MACHINE_CONFIG_KUBE_CLIENT_AGENT_IMAGE}" \ + --asset-input-dir=./assets/tls \ + --asset-output-dir=./assets/etcd-bootstrap \ + --config-output-file=./assets/etcd-bootstrap/config \ + --cluster-config-file="${INSTALLER_ASSETS_DIR}/manifests/cluster-network-02-config.yml" \ + --cluster-configmap-file="${INSTALLER_ASSETS_DIR}/manifests/cluster-config.yaml" \ + --infra-config-file="${INSTALLER_ASSETS_DIR}/manifests/cluster-infrastructure-02-config.yml" diff --git a/pkg/cmd/aio/aio.go b/pkg/cmd/aio/aio.go new file mode 100644 index 0000000000..d0ff94b2b6 --- /dev/null +++ b/pkg/cmd/aio/aio.go @@ -0,0 +1,203 @@ +package aio + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "strings" + + "github.com/openshift/cluster-etcd-operator/pkg/etcdenvvar" + "github.com/openshift/cluster-etcd-operator/pkg/operator/etcd_assets" + "github.com/openshift/cluster-etcd-operator/pkg/tlshelpers" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "k8s.io/klog" +) + +const ( + aioNodeName = "aio" + aioNodeInternalIP = "127.0.0.1" +) + +// aioOpts holds values to drive the aio command. +type aioOpts struct { + errOut io.Writer + + etcdCACert string + etcdCAKey string + etcdMetricCACert string + etcdMetricCAKey string + assetOutputDir string + etcdImage string +} + +// NewAIOCommand creates a all-in-one render command. +func NewAIOCommand(errOut io.Writer) *cobra.Command { + aioOpts := aioOpts{ + errOut: errOut, + } + cmd := &cobra.Command{ + Use: "aio", + Short: "Render all-in-one etcd manifests and related resources", + Run: func(cmd *cobra.Command, args []string) { + must := func(fn func() error) { + if err := fn(); err != nil { + if cmd.HasParent() { + klog.Fatal(err) + } + fmt.Fprint(aioOpts.errOut, err.Error()) + } + } + + must(aioOpts.Validate) + must(aioOpts.Run) + }, + } + + aioOpts.AddFlags(cmd.Flags()) + + return cmd +} + +func (a *aioOpts) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&a.etcdCACert, "etcd-ca-cert", a.etcdCACert, "path to etcd CA certificate") + fs.StringVar(&a.etcdCAKey, "etcd-ca-key", a.etcdCAKey, "path to etcd CA key") + fs.StringVar(&a.etcdMetricCACert, "etcd-metric-ca-cert", a.etcdMetricCACert, "path to etcd metric CA certificate") + fs.StringVar(&a.etcdMetricCAKey, "etcd-metric-ca-key", a.etcdMetricCAKey, "path to etcd metric CA key") + fs.StringVar(&a.assetOutputDir, "asset-output-dir", a.assetOutputDir, "path for rendered assets") + fs.StringVar(&a.etcdImage, "manifest-etcd-image", a.etcdImage, "etcd manifest image") +} + +// Validate verifies the inputs. +func (a *aioOpts) Validate() error { + if len(a.etcdCACert) == 0 { + return errors.New("missing required flag: --etcd-ca-cert") + } + if len(a.etcdCAKey) == 0 { + return errors.New("missing required flag: --etcd-ca-key") + } + if len(a.etcdMetricCACert) == 0 { + return errors.New("missing required flag: --etcd-metric-ca-cert") + } + if len(a.etcdMetricCAKey) == 0 { + return errors.New("missing required flag: --etcd-metric-ca-key") + } + if len(a.assetOutputDir) == 0 { + return errors.New("missing required flag: --asset-output-dir") + } + if len(a.etcdImage) == 0 { + return errors.New("missing required flag: --manifest-etcd-image") + } + return nil +} + +// Run contains the logic of the aio command. +func (a *aioOpts) Run() error { + err := a.generateEtcdNodeCerts(aioNodeName, aioNodeInternalIP) + if err != nil { + return err + } + return a.renderEtcdPod(aioNodeName, aioNodeInternalIP) +} + +func (a *aioOpts) renderEtcdPod(nodeName, nodeInternalIP string) error { + envVarMap, err := getAIOEtcdEnvVars(nodeName, nodeInternalIP, a.etcdImage) + if err != nil { + return fmt.Errorf("Failed to get all-in-one env variables for pod: %s", err) + } + + replacer, err := etcdenvvar.GetSubstitutionReplacer(envVarMap, a.etcdImage) + if err != nil { + return fmt.Errorf("Failed to render pod manifest: %s", err) + } + + podContent := string(etcd_assets.MustAsset("etcd/pod.yaml")) + podContent = replacer.Replace(podContent) + podContent = strings.ReplaceAll(podContent, "REVISION", "1") + podContent = strings.ReplaceAll(podContent, "NODE_NAME", nodeName) + podContent = strings.ReplaceAll(podContent, "NODE_ENVVAR_NAME", strings.ReplaceAll(strings.ReplaceAll(nodeName, "-", "_"), ".", "_")) + + err = ioutil.WriteFile(path.Join(a.assetOutputDir, "etcd-member.yaml"), []byte(podContent), 0644) + if err != nil { + return fmt.Errorf("Failed to write pod manifest: %s", err) + } + return nil +} + +func (a *aioOpts) generateEtcdNodeCerts(nodeName, nodeInternalIP string) error { + caCertData, err := ioutil.ReadFile(a.etcdCACert) + if err != nil { + return fmt.Errorf("Failed to read --etcd-ca-cert file: %s", err) + } + + caKeyData, err := ioutil.ReadFile(a.etcdCAKey) + if err != nil { + return fmt.Errorf("Failed to read --etcd-ca-key file: %s", err) + } + + metricCACertData, err := ioutil.ReadFile(a.etcdMetricCACert) + if err != nil { + return fmt.Errorf("Failed to read --etcd-metric-ca-cert file: %s", err) + } + + metricCAKeyData, err := ioutil.ReadFile(a.etcdMetricCAKey) + if err != nil { + return fmt.Errorf("Failed to read --etcd-metric-ca-key file: %s", err) + } + + nodeInternalIPs := []string{nodeInternalIP} + + certData, keyData, err := tlshelpers.CreateServerCertKey(caCertData, caKeyData, nodeInternalIPs) + if err != nil { + return err + } + err = a.writeCertKeyFiles(tlshelpers.EtcdAllServingSecretName, tlshelpers.GetServingSecretNameForNode(nodeName), certData, keyData) + if err != nil { + return err + } + + certData, keyData, err = tlshelpers.CreatePeerCertKey(caCertData, caKeyData, nodeInternalIPs) + if err != nil { + return err + } + err = a.writeCertKeyFiles(tlshelpers.EtcdAllPeerSecretName, tlshelpers.GetPeerClientSecretNameForNode(nodeName), certData, keyData) + if err != nil { + return err + } + + certData, keyData, err = tlshelpers.CreateMetricCertKey(metricCACertData, metricCAKeyData, nodeInternalIPs) + if err != nil { + return err + } + err = a.writeCertKeyFiles(tlshelpers.EtcdAllServingMetricsSecretName, tlshelpers.GetServingMetricsSecretNameForNode(nodeName), certData, keyData) + if err != nil { + return err + } + + return nil +} + +func (a *aioOpts) writeCertKeyFiles(allSecretName, nodeSecretName string, certData, keyData *bytes.Buffer) error { + dir := path.Join(a.assetOutputDir, "secrets", allSecretName) + + err := os.MkdirAll(dir, 0755) + if err != nil { + return fmt.Errorf("Failed to create %s directory: %s", allSecretName, err) + } + + err = ioutil.WriteFile(path.Join(dir, nodeSecretName+".crt"), certData.Bytes(), 0600) + if err != nil { + return fmt.Errorf("Failed to write %s cert: %s", allSecretName, err) + } + err = ioutil.WriteFile(path.Join(dir, nodeSecretName+".key"), keyData.Bytes(), 0600) + if err != nil { + return fmt.Errorf("Failed to write %s key: %s", allSecretName, err) + } + + return nil +} diff --git a/pkg/cmd/aio/env.go b/pkg/cmd/aio/env.go new file mode 100644 index 0000000000..c088058636 --- /dev/null +++ b/pkg/cmd/aio/env.go @@ -0,0 +1,29 @@ +package aio + +import ( + "fmt" + + "github.com/openshift/cluster-etcd-operator/pkg/etcdenvvar" +) + +func getAIOEtcdEnvVars(nodeName, nodeInternalIP, imagePullSpec string) (map[string]string, error) { + ret := map[string]string{ + "ETCDCTL_API": "3", + "ETCDCTL_CACERT": "/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt", + "ETCDCTL_CERT": "/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt", + "ETCDCTL_KEY": "/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key", + "ETCDCTL_ENDPOINTS": fmt.Sprintf("https://%s:2379", nodeInternalIP), + "ALL_ETCD_ENDPOINTS": fmt.Sprintf("https://%s:2379", nodeInternalIP), + "ETCD_IMAGE": imagePullSpec, + "ETCD_HEARTBEAT_INTERVAL": "100", // etcd default + "ETCD_ELECTION_TIMEOUT": "1000", // etcd default + + fmt.Sprintf("NODE_%s_ETCD_NAME", nodeName): nodeName, + fmt.Sprintf("NODE_%s_IP", nodeName): nodeInternalIP, + fmt.Sprintf("NODE_%s_ETCD_URL_HOST", nodeName): nodeInternalIP, + } + for k, v := range etcdenvvar.FixedEtcdEnvVars { + ret[k] = v + } + return ret, nil +} diff --git a/pkg/etcdenvvar/replacer.go b/pkg/etcdenvvar/replacer.go new file mode 100644 index 0000000000..426a517bc1 --- /dev/null +++ b/pkg/etcdenvvar/replacer.go @@ -0,0 +1,28 @@ +package etcdenvvar + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/util/sets" +) + +func GetSubstitutionReplacer(envVarMap map[string]string, imagePullSpec string) (*strings.Replacer, error) { + if len(envVarMap) == 0 { + return nil, fmt.Errorf("missing env var values") + } + + envVarLines := []string{} + for _, k := range sets.StringKeySet(envVarMap).List() { + v := envVarMap[k] + envVarLines = append(envVarLines, fmt.Sprintf(" - name: %q", k)) + envVarLines = append(envVarLines, fmt.Sprintf(" value: %q", v)) + } + + return strings.NewReplacer( + "${IMAGE}", imagePullSpec, + "${LISTEN_ON_ALL_IPS}", "0.0.0.0", // TODO this needs updating to detect ipv6-ness + "${LOCALHOST_IP}", "127.0.0.1", // TODO this needs updating to detect ipv6-ness + "${COMPUTED_ENV_VARS}", strings.Join(envVarLines, "\n"), // lacks beauty, but it works + ), nil +} diff --git a/pkg/operator/etcdcertsigner/etcdcertsignercontroller.go b/pkg/operator/etcdcertsigner/etcdcertsignercontroller.go index a88e3a7066..61eabb6daf 100644 --- a/pkg/operator/etcdcertsigner/etcdcertsignercontroller.go +++ b/pkg/operator/etcdcertsigner/etcdcertsignercontroller.go @@ -3,18 +3,12 @@ package etcdcertsigner import ( "bytes" "context" - "crypto/x509" - "crypto/x509/pkix" - "errors" - "fmt" - "strings" "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" @@ -23,20 +17,13 @@ import ( configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" configv1listers "github.com/openshift/client-go/config/listers/config/v1" "github.com/openshift/library-go/pkg/controller/factory" - "github.com/openshift/library-go/pkg/crypto" "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/openshift/library-go/pkg/operator/v1helpers" "github.com/openshift/cluster-etcd-operator/pkg/dnshelpers" "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" -) - -const ( - EtcdCertValidity = 3 * 365 * 24 * time.Hour - peerOrg = "system:etcd-peers" - serverOrg = "system:etcd-servers" - metricOrg = "system:etcd-metrics" + "github.com/openshift/cluster-etcd-operator/pkg/tlshelpers" ) type EtcdCertSignerController struct { @@ -127,43 +114,47 @@ func (c *EtcdCertSignerController) syncAllMasters(recorder events.Recorder) erro // build the combined secrets that we're going to install combinedPeerSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.TargetNamespace, Name: "etcd-all-peer"}, + ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.TargetNamespace, Name: tlshelpers.EtcdAllPeerSecretName}, Type: corev1.SecretTypeOpaque, Data: map[string][]byte{}, } combinedServingSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.TargetNamespace, Name: "etcd-all-serving"}, + ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.TargetNamespace, Name: tlshelpers.EtcdAllServingSecretName}, Type: corev1.SecretTypeOpaque, Data: map[string][]byte{}, } combinedServingMetricsSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.TargetNamespace, Name: "etcd-all-serving-metrics"}, + ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.TargetNamespace, Name: tlshelpers.EtcdAllServingMetricsSecretName}, Type: corev1.SecretTypeOpaque, Data: map[string][]byte{}, } for _, node := range nodes { - currPeer, err := c.secretLister.Secrets(operatorclient.TargetNamespace).Get(getPeerClientSecretNameForNode(node)) + peerSecretName := tlshelpers.GetPeerClientSecretNameForNode(node.Name) + servingSecretName := tlshelpers.GetServingSecretNameForNode(node.Name) + servingMetricsSecretName := tlshelpers.GetServingMetricsSecretNameForNode(node.Name) + + currPeer, err := c.secretLister.Secrets(operatorclient.TargetNamespace).Get(peerSecretName) if err != nil { errs = append(errs, err) } else { - combinedPeerSecret.Data[getPeerClientSecretNameForNode(node)+".crt"] = currPeer.Data["tls.crt"] - combinedPeerSecret.Data[getPeerClientSecretNameForNode(node)+".key"] = currPeer.Data["tls.key"] + combinedPeerSecret.Data[peerSecretName+".crt"] = currPeer.Data["tls.crt"] + combinedPeerSecret.Data[peerSecretName+".key"] = currPeer.Data["tls.key"] } - currServing, err := c.secretLister.Secrets(operatorclient.TargetNamespace).Get(getServingSecretNameForNode(node)) + currServing, err := c.secretLister.Secrets(operatorclient.TargetNamespace).Get(servingSecretName) if err != nil { errs = append(errs, err) } else { - combinedServingSecret.Data[getServingSecretNameForNode(node)+".crt"] = currServing.Data["tls.crt"] - combinedServingSecret.Data[getServingSecretNameForNode(node)+".key"] = currServing.Data["tls.key"] + combinedServingSecret.Data[servingSecretName+".crt"] = currServing.Data["tls.crt"] + combinedServingSecret.Data[servingSecretName+".key"] = currServing.Data["tls.key"] } - currServingMetrics, err := c.secretLister.Secrets(operatorclient.TargetNamespace).Get(getServingMetricsSecretNameForNode(node)) + currServingMetrics, err := c.secretLister.Secrets(operatorclient.TargetNamespace).Get(servingMetricsSecretName) if err != nil { errs = append(errs, err) } else { - combinedServingMetricsSecret.Data[getServingMetricsSecretNameForNode(node)+".crt"] = currServingMetrics.Data["tls.crt"] - combinedServingMetricsSecret.Data[getServingMetricsSecretNameForNode(node)+".key"] = currServingMetrics.Data["tls.key"] + combinedServingMetricsSecret.Data[servingMetricsSecretName+".crt"] = currServingMetrics.Data["tls.crt"] + combinedServingMetricsSecret.Data[servingMetricsSecretName+".key"] = currServingMetrics.Data["tls.key"] } } if len(errs) > 0 { @@ -187,20 +178,10 @@ func (c *EtcdCertSignerController) syncAllMasters(recorder events.Recorder) erro return utilerrors.NewAggregate(errs) } -func getPeerClientSecretNameForNode(node *corev1.Node) string { - return fmt.Sprintf("etcd-peer-%s", node.Name) -} -func getServingSecretNameForNode(node *corev1.Node) string { - return fmt.Sprintf("etcd-serving-%s", node.Name) -} -func getServingMetricsSecretNameForNode(node *corev1.Node) string { - return fmt.Sprintf("etcd-serving-metrics-%s", node.Name) -} - func (c *EtcdCertSignerController) createSecretForNode(node *corev1.Node, recorder events.Recorder) error { - etcdPeerClientCertName := getPeerClientSecretNameForNode(node) - etcdServingCertName := getServingSecretNameForNode(node) - metricsServingCertName := getServingMetricsSecretNameForNode(node) + etcdPeerClientCertName := tlshelpers.GetPeerClientSecretNameForNode(node.Name) + etcdServingCertName := tlshelpers.GetServingSecretNameForNode(node.Name) + metricsServingCertName := tlshelpers.GetServingMetricsSecretNameForNode(node.Name) var err error _, err = c.secretLister.Secrets(operatorclient.TargetNamespace).Get(etcdPeerClientCertName) @@ -229,32 +210,19 @@ func (c *EtcdCertSignerController) createSecretForNode(node *corev1.Node, record if err != nil { return err } - peerHostNames := append([]string{"localhost"}, nodeInternalIPs...) - serverHostNames := append([]string{ - "localhost", - "etcd.kube-system.svc", - "etcd.kube-system.svc.cluster.local", - "etcd.openshift-etcd.svc", - "etcd.openshift-etcd.svc.cluster.local", - "127.0.0.1", - "::1", - "0:0:0:0:0:0:0:1", - }, nodeInternalIPs...) - // TODO debt left for @hexfusion or @sanchezl - fakePodFQDN := "etcd-client" // create the certificates and update them in the API - pCert, pKey, err := createNewCombinedClientAndServingCerts(etcdCASecret.Data["tls.crt"], etcdCASecret.Data["tls.key"], fakePodFQDN, peerOrg, peerHostNames) + pCert, pKey, err := tlshelpers.CreatePeerCertKey(etcdCASecret.Data["tls.crt"], etcdCASecret.Data["tls.key"], nodeInternalIPs) err = c.createSecret(etcdPeerClientCertName, pCert, pKey, recorder) if err != nil { return err } - sCert, sKey, err := createNewCombinedClientAndServingCerts(etcdCASecret.Data["tls.crt"], etcdCASecret.Data["tls.key"], fakePodFQDN, serverOrg, serverHostNames) + sCert, sKey, err := tlshelpers.CreateServerCertKey(etcdCASecret.Data["tls.crt"], etcdCASecret.Data["tls.key"], nodeInternalIPs) err = c.createSecret(etcdServingCertName, sCert, sKey, recorder) if err != nil { return err } - metricCert, metricKey, err := createNewCombinedClientAndServingCerts(etcdMetricCASecret.Data["tls.crt"], etcdMetricCASecret.Data["tls.key"], fakePodFQDN, metricOrg, serverHostNames) + metricCert, metricKey, err := tlshelpers.CreateMetricCertKey(etcdMetricCASecret.Data["tls.crt"], etcdMetricCASecret.Data["tls.key"], nodeInternalIPs) err = c.createSecret(metricsServingCertName, metricCert, metricKey, recorder) if err != nil { return err @@ -263,74 +231,6 @@ func (c *EtcdCertSignerController) createSecretForNode(node *corev1.Node, record return nil } -func createNewCombinedClientAndServingCerts(caCert, caKey []byte, podFQDN, org string, peerHostNames []string) (*bytes.Buffer, *bytes.Buffer, error) { - cn, err := getCommonNameFromOrg(org) - etcdCAKeyPair, err := crypto.GetCAFromBytes(caCert, caKey) - if err != nil { - return nil, nil, err - } - - certConfig, err := etcdCAKeyPair.MakeServerCertForDuration(sets.NewString(peerHostNames...), EtcdCertValidity, func(cert *x509.Certificate) error { - - cert.Issuer = pkix.Name{ - OrganizationalUnit: []string{"openshift"}, - CommonName: cn, - } - cert.Subject = pkix.Name{ - Organization: []string{org}, - CommonName: strings.TrimSuffix(org, "s") + ":" + podFQDN, - } - cert.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth} - - // TODO: Extended Key Usage: - // All profiles expect a x509.ExtKeyUsageCodeSigning set on extended Key Usages - // need to investigage: https://github.com/etcd-io/etcd/issues/9398#issuecomment-435340312 - // TODO: some extensions are missing form cfssl. - // e.g. - // X509v3 Subject Key Identifier: - // B7:30:0B:CF:47:4E:21:AE:13:60:74:42:B0:D9:C4:F3:26:69:63:03 - // X509v3 Authority Key Identifier: - // keyid:9B:C0:6B:0C:8E:5C:73:6A:83:B1:E4:54:97:D3:62:18:8A:9C:BC:1E - // TODO: Change serial number logic, to something as follows. - // The following is taken from CFSSL library. - // If CFSSL is providing the serial numbers, it makes - // sense to use the max supported size. - - // serialNumber := make([]byte, 20) - // _, err = io.ReadFull(rand.Reader, serialNumber) - // if err != nil { - // return err - // } - // - // // SetBytes interprets buf as the bytes of a big-endian - // // unsigned integer. The leading byte should be masked - // // off to ensure it isn't negative. - // serialNumber[0] &= 0x7F - // cert.SerialNumber = new(big.Int).SetBytes(serialNumber) - return nil - }) - if err != nil { - return nil, nil, err - } - - certBytes := &bytes.Buffer{} - keyBytes := &bytes.Buffer{} - if err := certConfig.WriteCertConfig(certBytes, keyBytes); err != nil { - return nil, nil, err - } - return certBytes, keyBytes, nil -} - -func getCommonNameFromOrg(org string) (string, error) { - if strings.Contains(org, "peer") || strings.Contains(org, "server") { - return "etcd-signer", nil - } - if strings.Contains(org, "metric") { - return "etcd-metric-signer", nil - } - return "", errors.New("unable to recognise secret name") -} - func (c *EtcdCertSignerController) createSecret(secretName string, cert *bytes.Buffer, key *bytes.Buffer, recorder events.Recorder) error { //TODO: Update annotations Not Before and Not After for Cert Rotation _, _, err := resourceapply.ApplySecret(c.secretClient, recorder, &corev1.Secret{ diff --git a/pkg/operator/targetconfigcontroller/targetconfigcontroller.go b/pkg/operator/targetconfigcontroller/targetconfigcontroller.go index 767bd6be9d..d0b5717b09 100644 --- a/pkg/operator/targetconfigcontroller/targetconfigcontroller.go +++ b/pkg/operator/targetconfigcontroller/targetconfigcontroller.go @@ -8,7 +8,6 @@ import ( corev1 "k8s.io/api/core/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1" @@ -111,7 +110,7 @@ func (c TargetConfigController) sync(ctx context.Context, syncCtx factory.SyncCo func createTargetConfig(c TargetConfigController, recorder events.Recorder, operatorSpec *operatorv1.StaticPodOperatorSpec) (bool, error) { errors := []error{} - contentReplacer, err := c.getSubstitutionReplacer(operatorSpec, c.targetImagePullSpec, c.operatorImagePullSpec) + contentReplacer, err := etcdenvvar.GetSubstitutionReplacer(c.envVarGetter.GetEnvVars(), c.targetImagePullSpec) if err != nil { return false, err } @@ -186,29 +185,6 @@ func loglevelToKlog(logLevel operatorv1.LogLevel) string { } } -func (c *TargetConfigController) getSubstitutionReplacer(operatorSpec *operatorv1.StaticPodOperatorSpec, imagePullSpec, operatorImagePullSpec string) (*strings.Replacer, error) { - envVarMap := c.envVarGetter.GetEnvVars() - if len(envVarMap) == 0 { - return nil, fmt.Errorf("missing env var values") - } - - envVarLines := []string{} - for _, k := range sets.StringKeySet(envVarMap).List() { - v := envVarMap[k] - envVarLines = append(envVarLines, fmt.Sprintf(" - name: %q", k)) - envVarLines = append(envVarLines, fmt.Sprintf(" value: %q", v)) - } - - return strings.NewReplacer( - "${IMAGE}", imagePullSpec, - "${OPERATOR_IMAGE}", operatorImagePullSpec, - "${VERBOSITY}", loglevelToKlog(operatorSpec.LogLevel), - "${LISTEN_ON_ALL_IPS}", "0.0.0.0", // TODO this needs updating to detect ipv6-ness - "${LOCALHOST_IP}", "127.0.0.1", // TODO this needs updating to detect ipv6-ness - "${COMPUTED_ENV_VARS}", strings.Join(envVarLines, "\n"), // lacks beauty, but it works - ), nil -} - func (c *TargetConfigController) manageRecoveryPod(substitutionReplacer *strings.Replacer, client coreclientv1.ConfigMapsGetter, recorder events.Recorder, operatorSpec *operatorv1.StaticPodOperatorSpec) (*corev1.ConfigMap, bool, error) { podBytes := etcd_assets.MustAsset("etcd/restore-pod.yaml") substitutedPodString := substitutionReplacer.Replace(string(podBytes)) diff --git a/pkg/tlshelpers/etcdcertsigner.go b/pkg/tlshelpers/etcdcertsigner.go new file mode 100644 index 0000000000..0fcad45882 --- /dev/null +++ b/pkg/tlshelpers/etcdcertsigner.go @@ -0,0 +1,137 @@ +package tlshelpers + +import ( + "bytes" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "fmt" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/openshift/library-go/pkg/crypto" +) + +const ( + etcdCertValidity = 3 * 365 * 24 * time.Hour + + peerOrg = "system:etcd-peers" + serverOrg = "system:etcd-servers" + metricOrg = "system:etcd-metrics" + + // TODO debt left for @hexfusion or @sanchezl + fakePodFQDN = "etcd-client" + + EtcdAllPeerSecretName = "etcd-all-peer" + EtcdAllServingSecretName = "etcd-all-serving" + EtcdAllServingMetricsSecretName = "etcd-all-serving-metrics" +) + +func GetPeerClientSecretNameForNode(nodeName string) string { + return fmt.Sprintf("etcd-peer-%s", nodeName) +} +func GetServingSecretNameForNode(nodeName string) string { + return fmt.Sprintf("etcd-serving-%s", nodeName) +} +func GetServingMetricsSecretNameForNode(nodeName string) string { + return fmt.Sprintf("etcd-serving-metrics-%s", nodeName) +} + +func getPeerHostNames(nodeInternalIPs []string) []string { + return append([]string{"localhost"}, nodeInternalIPs...) +} + +func getServerHostNames(nodeInternalIPs []string) []string { + return append([]string{ + "localhost", + "etcd.kube-system.svc", + "etcd.kube-system.svc.cluster.local", + "etcd.openshift-etcd.svc", + "etcd.openshift-etcd.svc.cluster.local", + "127.0.0.1", + "::1", + "0:0:0:0:0:0:0:1", + }, nodeInternalIPs...) +} + +func CreatePeerCertKey(caCert, caKey []byte, nodeInternalIPs []string) (*bytes.Buffer, *bytes.Buffer, error) { + return createNewCombinedClientAndServingCerts(caCert, caKey, fakePodFQDN, peerOrg, getPeerHostNames(nodeInternalIPs)) +} + +func CreateServerCertKey(caCert, caKey []byte, nodeInternalIPs []string) (*bytes.Buffer, *bytes.Buffer, error) { + return createNewCombinedClientAndServingCerts(caCert, caKey, fakePodFQDN, serverOrg, getServerHostNames(nodeInternalIPs)) +} + +func CreateMetricCertKey(caCert, caKey []byte, nodeInternalIPs []string) (*bytes.Buffer, *bytes.Buffer, error) { + return createNewCombinedClientAndServingCerts(caCert, caKey, fakePodFQDN, metricOrg, getServerHostNames(nodeInternalIPs)) +} + +func createNewCombinedClientAndServingCerts(caCert, caKey []byte, podFQDN, org string, hostNames []string) (*bytes.Buffer, *bytes.Buffer, error) { + cn, err := getCommonNameFromOrg(org) + etcdCAKeyPair, err := crypto.GetCAFromBytes(caCert, caKey) + if err != nil { + return nil, nil, err + } + + certConfig, err := etcdCAKeyPair.MakeServerCertForDuration(sets.NewString(hostNames...), etcdCertValidity, func(cert *x509.Certificate) error { + + cert.Issuer = pkix.Name{ + OrganizationalUnit: []string{"openshift"}, + CommonName: cn, + } + cert.Subject = pkix.Name{ + Organization: []string{org}, + CommonName: strings.TrimSuffix(org, "s") + ":" + podFQDN, + } + cert.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth} + + // TODO: Extended Key Usage: + // All profiles expect a x509.ExtKeyUsageCodeSigning set on extended Key Usages + // need to investigage: https://github.com/etcd-io/etcd/issues/9398#issuecomment-435340312 + // TODO: some extensions are missing form cfssl. + // e.g. + // X509v3 Subject Key Identifier: + // B7:30:0B:CF:47:4E:21:AE:13:60:74:42:B0:D9:C4:F3:26:69:63:03 + // X509v3 Authority Key Identifier: + // keyid:9B:C0:6B:0C:8E:5C:73:6A:83:B1:E4:54:97:D3:62:18:8A:9C:BC:1E + // TODO: Change serial number logic, to something as follows. + // The following is taken from CFSSL library. + // If CFSSL is providing the serial numbers, it makes + // sense to use the max supported size. + + // serialNumber := make([]byte, 20) + // _, err = io.ReadFull(rand.Reader, serialNumber) + // if err != nil { + // return err + // } + // + // // SetBytes interprets buf as the bytes of a big-endian + // // unsigned integer. The leading byte should be masked + // // off to ensure it isn't negative. + // serialNumber[0] &= 0x7F + // cert.SerialNumber = new(big.Int).SetBytes(serialNumber) + return nil + }) + if err != nil { + return nil, nil, err + } + + certBytes := &bytes.Buffer{} + keyBytes := &bytes.Buffer{} + if err := certConfig.WriteCertConfig(certBytes, keyBytes); err != nil { + return nil, nil, err + } + return certBytes, keyBytes, nil +} + +func getCommonNameFromOrg(org string) (string, error) { + if strings.Contains(org, "peer") || strings.Contains(org, "server") { + return "etcd-signer", nil + } + if strings.Contains(org, "metric") { + return "etcd-metric-signer", nil + } + return "", errors.New("unable to recognise secret name") +}