WIP: experiment with a render command for an all-in-one config

cmd/cluster-etcd-operator/main.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/openshift/cluster-etcd-operator/pkg/cmd/aio"
 	operatorcmd "github.com/openshift/cluster-etcd-operator/pkg/cmd/operator"
 	"github.com/openshift/cluster-etcd-operator/pkg/cmd/render"
 	"github.com/openshift/cluster-etcd-operator/pkg/cmd/waitforceo"
@@ -60,6 +61,7 @@ func NewSSCSCommand() *cobra.Command {
 	cmd.AddCommand(prune.NewPrune())
 	cmd.AddCommand(certsyncpod.NewCertSyncControllerCommand(operator.CertConfigMaps, operator.CertSecrets))
 	cmd.AddCommand(waitforceo.NewWaitForCeoCommand(os.Stderr))
+	cmd.AddCommand(aio.NewAIOCommand(os.Stderr))
 
 	return cmd
 }

hack/aio-render.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# openshift-install --dir test/ create aio-config
+
+INSTALLER_ASSETS_DIR="$1"
+IGNITION_CONFIG="${INSTALLER_ASSETS_DIR}/aio.ign"
+
+# oc adm release info registry.svc.ci.openshift.org/ocp/release:4.6.0-0.ci-2020-07-21-114552 -o json > release:4.6.0-0.ci-2020-07-21-114552-info.json
+RELEASE_INFO="release:4.6.0-0.ci-2020-07-21-114552-info.json"
+
+mkdir -p ./assets/tls
+
+# Unpack the TLS assets from the ignition file
+jq  -c '.storage.files[] | {p:.path,c:.contents.source}' "${IGNITION_CONFIG}" | while read f t; do
+    p=$(echo $f | jq -r .p)
+    c=$(echo $f | jq -r .c)
+
+    [[ "$p" != /opt/openshift/tls/* ]] && continue
+
+    echo "${c#data:text/plain;charset=utf-8;base64,}" | base64 -d > "./assets/tls/$(basename $p)"
+done
+
+image_for() {
+    jq -r '.references.spec.tags[] | select(.name =="tools") | .from.name' "${RELEASE_INFO}"
+}
+
+MACHINE_CONFIG_ETCD_IMAGE=$(image_for etcd)
+
+./cluster-etcd-operator aio \
+    --etcd-ca-cert=./assets/tls/etcd-signer.crt \
+    --etcd-ca-key=./assets/tls/etcd-signer.key \
+    --etcd-metric-ca-cert=./assets/tls/etcd-metric-signer.crt \
+    --etcd-metric-ca-key=./assets/tls/etcd-metric-signer.key \
+    --asset-output-dir=./assets/etcd-aio \
+    --manifest-etcd-image="${MACHINE_CONFIG_ETCD_IMAGE}"

hack/render.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# openshift-install --dir test/ create manifests
+# openshift-install --dir test/ create aio-config
+
+INSTALLER_ASSETS_DIR="$1"
+IGNITION_CONFIG="${INSTALLER_ASSETS_DIR}/aio.ign"
+
+CLUSTER_DOMAIN="markmc.devcluster.openshift.com"
+
+# oc adm release info registry.svc.ci.openshift.org/ocp/release:4.6.0-0.ci-2020-07-21-114552 -o json > release:4.6.0-0.ci-2020-07-21-114552-info.json
+RELEASE_INFO="release:4.6.0-0.ci-2020-07-21-114552-info.json"
+
+mkdir -p ./assets/tls
+
+# Unpack the TLS assets from the ignition file
+jq  -c '.storage.files[] | {p:.path,c:.contents.source}' "${IGNITION_CONFIG}" | while read f t; do
+    p=$(echo $f | jq -r .p)
+    c=$(echo $f | jq -r .c)
+
+    [[ "$p" != /opt/openshift/tls/* ]] && continue
+
+    echo "${c#data:text/plain;charset=utf-8;base64,}" | base64 -d > "./assets/tls/$(basename $p)"
+done
+
+image_for() {
+    jq -r '.references.spec.tags[] | select(.name =="tools") | .from.name' "${RELEASE_INFO}"
+}
+
+MACHINE_CONFIG_ETCD_IMAGE=$(image_for etcd)
+CLUSTER_ETCD_OPERATOR_IMAGE=$(image_for cluster-etcd-operator)
+MACHINE_CONFIG_OPERATOR_IMAGE=$(image_for machine-config-operator)
+MACHINE_CONFIG_KUBE_CLIENT_AGENT_IMAGE=$(image_for kube-client-agent)
+
+./cluster-etcd-operator render \
+    --templates-input-dir=./bindata/bootkube \
+    --etcd-ca=./assets/tls/etcd-ca-bundle.crt \
+    --etcd-metric-ca=./assets/tls/etcd-metric-ca-bundle.crt \
+    --manifest-etcd-image="${MACHINE_CONFIG_ETCD_IMAGE}" \
+    --etcd-discovery-domain="${CLUSTER_DOMAIN}" \
+    --manifest-cluster-etcd-operator-image="${CLUSTER_ETCD_OPERATOR_IMAGE}" \
+    --manifest-setup-etcd-env-image="${MACHINE_CONFIG_OPERATOR_IMAGE}" \
+    --manifest-kube-client-agent-image="${MACHINE_CONFIG_KUBE_CLIENT_AGENT_IMAGE}" \
+    --asset-input-dir=./assets/tls \
+    --asset-output-dir=./assets/etcd-bootstrap \
+    --config-output-file=./assets/etcd-bootstrap/config \
+    --cluster-config-file="${INSTALLER_ASSETS_DIR}/manifests/cluster-network-02-config.yml" \
+    --cluster-configmap-file="${INSTALLER_ASSETS_DIR}/manifests/cluster-config.yaml" \
+    --infra-config-file="${INSTALLER_ASSETS_DIR}/manifests/cluster-infrastructure-02-config.yml"

pkg/cmd/aio/aio.go

@@ -0,0 +1,203 @@
+package aio
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/openshift/cluster-etcd-operator/pkg/etcdenvvar"
+	"github.com/openshift/cluster-etcd-operator/pkg/operator/etcd_assets"
+	"github.com/openshift/cluster-etcd-operator/pkg/tlshelpers"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"k8s.io/klog"
+)
+
+const (
+	aioNodeName       = "aio"
+	aioNodeInternalIP = "127.0.0.1"
+)
+
+// aioOpts holds values to drive the aio command.
+type aioOpts struct {
+	errOut io.Writer
+
+	etcdCACert       string
+	etcdCAKey        string
+	etcdMetricCACert string
+	etcdMetricCAKey  string
+	assetOutputDir   string
+	etcdImage        string
+}
+
+// NewAIOCommand creates a all-in-one render command.
+func NewAIOCommand(errOut io.Writer) *cobra.Command {
+	aioOpts := aioOpts{
+		errOut: errOut,
+	}
+	cmd := &cobra.Command{
+		Use:   "aio",
+		Short: "Render all-in-one etcd manifests and related resources",
+		Run: func(cmd *cobra.Command, args []string) {
+			must := func(fn func() error) {
+				if err := fn(); err != nil {
+					if cmd.HasParent() {
+						klog.Fatal(err)
+					}
+					fmt.Fprint(aioOpts.errOut, err.Error())
+				}
+			}
+
+			must(aioOpts.Validate)
+			must(aioOpts.Run)
+		},
+	}
+
+	aioOpts.AddFlags(cmd.Flags())
+
+	return cmd
+}
+
+func (a *aioOpts) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&a.etcdCACert, "etcd-ca-cert", a.etcdCACert, "path to etcd CA certificate")
+	fs.StringVar(&a.etcdCAKey, "etcd-ca-key", a.etcdCAKey, "path to etcd CA key")
+	fs.StringVar(&a.etcdMetricCACert, "etcd-metric-ca-cert", a.etcdMetricCACert, "path to etcd metric CA certificate")
+	fs.StringVar(&a.etcdMetricCAKey, "etcd-metric-ca-key", a.etcdMetricCAKey, "path to etcd metric CA key")
+	fs.StringVar(&a.assetOutputDir, "asset-output-dir", a.assetOutputDir, "path for rendered assets")
+	fs.StringVar(&a.etcdImage, "manifest-etcd-image", a.etcdImage, "etcd manifest image")
+}
+
+// Validate verifies the inputs.
+func (a *aioOpts) Validate() error {
+	if len(a.etcdCACert) == 0 {
+		return errors.New("missing required flag: --etcd-ca-cert")
+	}
+	if len(a.etcdCAKey) == 0 {
+		return errors.New("missing required flag: --etcd-ca-key")
+	}
+	if len(a.etcdMetricCACert) == 0 {
+		return errors.New("missing required flag: --etcd-metric-ca-cert")
+	}
+	if len(a.etcdMetricCAKey) == 0 {
+		return errors.New("missing required flag: --etcd-metric-ca-key")
+	}
+	if len(a.assetOutputDir) == 0 {
+		return errors.New("missing required flag: --asset-output-dir")
+	}
+	if len(a.etcdImage) == 0 {
+		return errors.New("missing required flag: --manifest-etcd-image")
+	}
+	return nil
+}
+
+// Run contains the logic of the aio command.
+func (a *aioOpts) Run() error {
+	err := a.generateEtcdNodeCerts(aioNodeName, aioNodeInternalIP)
+	if err != nil {
+		return err
+	}
+	return a.renderEtcdPod(aioNodeName, aioNodeInternalIP)
+}
+
+func (a *aioOpts) renderEtcdPod(nodeName, nodeInternalIP string) error {
+	envVarMap, err := getAIOEtcdEnvVars(nodeName, nodeInternalIP, a.etcdImage)
+	if err != nil {
+		return fmt.Errorf("Failed to get all-in-one env variables for pod: %s", err)
+	}
+
+	replacer, err := etcdenvvar.GetSubstitutionReplacer(envVarMap, a.etcdImage)
+	if err != nil {
+		return fmt.Errorf("Failed to render pod manifest: %s", err)
+	}
+
+	podContent := string(etcd_assets.MustAsset("etcd/pod.yaml"))
+	podContent = replacer.Replace(podContent)
+	podContent = strings.ReplaceAll(podContent, "REVISION", "1")
+	podContent = strings.ReplaceAll(podContent, "NODE_NAME", nodeName)
+	podContent = strings.ReplaceAll(podContent, "NODE_ENVVAR_NAME", strings.ReplaceAll(strings.ReplaceAll(nodeName, "-", "_"), ".", "_"))
+
+	err = ioutil.WriteFile(path.Join(a.assetOutputDir, "etcd-member.yaml"), []byte(podContent), 0644)
+	if err != nil {
+		return fmt.Errorf("Failed to write pod manifest: %s", err)
+	}
+	return nil
+}
+
+func (a *aioOpts) generateEtcdNodeCerts(nodeName, nodeInternalIP string) error {
+	caCertData, err := ioutil.ReadFile(a.etcdCACert)
+	if err != nil {
+		return fmt.Errorf("Failed to read --etcd-ca-cert file: %s", err)
+	}
+
+	caKeyData, err := ioutil.ReadFile(a.etcdCAKey)
+	if err != nil {
+		return fmt.Errorf("Failed to read --etcd-ca-key file: %s", err)
+	}
+
+	metricCACertData, err := ioutil.ReadFile(a.etcdMetricCACert)
+	if err != nil {
+		return fmt.Errorf("Failed to read --etcd-metric-ca-cert file: %s", err)
+	}
+
+	metricCAKeyData, err := ioutil.ReadFile(a.etcdMetricCAKey)
+	if err != nil {
+		return fmt.Errorf("Failed to read --etcd-metric-ca-key file: %s", err)
+	}
+
+	nodeInternalIPs := []string{nodeInternalIP}
+
+	certData, keyData, err := tlshelpers.CreateServerCertKey(caCertData, caKeyData, nodeInternalIPs)
+	if err != nil {
+		return err
+	}
+	err = a.writeCertKeyFiles(tlshelpers.EtcdAllServingSecretName, tlshelpers.GetServingSecretNameForNode(nodeName), certData, keyData)
+	if err != nil {
+		return err
+	}
+
+	certData, keyData, err = tlshelpers.CreatePeerCertKey(caCertData, caKeyData, nodeInternalIPs)
+	if err != nil {
+		return err
+	}
+	err = a.writeCertKeyFiles(tlshelpers.EtcdAllPeerSecretName, tlshelpers.GetPeerClientSecretNameForNode(nodeName), certData, keyData)
+	if err != nil {
+		return err
+	}
+
+	certData, keyData, err = tlshelpers.CreateMetricCertKey(metricCACertData, metricCAKeyData, nodeInternalIPs)
+	if err != nil {
+		return err
+	}
+	err = a.writeCertKeyFiles(tlshelpers.EtcdAllServingMetricsSecretName, tlshelpers.GetServingMetricsSecretNameForNode(nodeName), certData, keyData)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (a *aioOpts) writeCertKeyFiles(allSecretName, nodeSecretName string, certData, keyData *bytes.Buffer) error {
+	dir := path.Join(a.assetOutputDir, "secrets", allSecretName)
+
+	err := os.MkdirAll(dir, 0755)
+	if err != nil {
+		return fmt.Errorf("Failed to create %s directory: %s", allSecretName, err)
+	}
+
+	err = ioutil.WriteFile(path.Join(dir, nodeSecretName+".crt"), certData.Bytes(), 0600)
+	if err != nil {
+		return fmt.Errorf("Failed to write %s cert: %s", allSecretName, err)
+	}
+	err = ioutil.WriteFile(path.Join(dir, nodeSecretName+".key"), keyData.Bytes(), 0600)
+	if err != nil {
+		return fmt.Errorf("Failed to write %s key: %s", allSecretName, err)
+	}
+
+	return nil
+}

pkg/cmd/aio/env.go

@@ -0,0 +1,29 @@
+package aio
+
+import (
+	"fmt"
+
+	"github.com/openshift/cluster-etcd-operator/pkg/etcdenvvar"
+)
+
+func getAIOEtcdEnvVars(nodeName, nodeInternalIP, imagePullSpec string) (map[string]string, error) {
+	ret := map[string]string{
+		"ETCDCTL_API":             "3",
+		"ETCDCTL_CACERT":          "/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt",
+		"ETCDCTL_CERT":            "/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt",
+		"ETCDCTL_KEY":             "/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key",
+		"ETCDCTL_ENDPOINTS":       fmt.Sprintf("https://%s:2379", nodeInternalIP),
+		"ALL_ETCD_ENDPOINTS":      fmt.Sprintf("https://%s:2379", nodeInternalIP),
+		"ETCD_IMAGE":              imagePullSpec,
+		"ETCD_HEARTBEAT_INTERVAL": "100",  // etcd default
+		"ETCD_ELECTION_TIMEOUT":   "1000", // etcd default
+
+		fmt.Sprintf("NODE_%s_ETCD_NAME", nodeName):     nodeName,
+		fmt.Sprintf("NODE_%s_IP", nodeName):            nodeInternalIP,
+		fmt.Sprintf("NODE_%s_ETCD_URL_HOST", nodeName): nodeInternalIP,
+	}
+	for k, v := range etcdenvvar.FixedEtcdEnvVars {
+		ret[k] = v
+	}
+	return ret, nil
+}

pkg/etcdenvvar/replacer.go

@@ -0,0 +1,28 @@
+package etcdenvvar
+
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+func GetSubstitutionReplacer(envVarMap map[string]string, imagePullSpec string) (*strings.Replacer, error) {
+	if len(envVarMap) == 0 {
+		return nil, fmt.Errorf("missing env var values")
+	}
+
+	envVarLines := []string{}
+	for _, k := range sets.StringKeySet(envVarMap).List() {
+		v := envVarMap[k]
+		envVarLines = append(envVarLines, fmt.Sprintf("      - name: %q", k))
+		envVarLines = append(envVarLines, fmt.Sprintf("        value: %q", v))
+	}
+
+	return strings.NewReplacer(
+		"${IMAGE}", imagePullSpec,
+		"${LISTEN_ON_ALL_IPS}", "0.0.0.0", // TODO this needs updating to detect ipv6-ness
+		"${LOCALHOST_IP}", "127.0.0.1", // TODO this needs updating to detect ipv6-ness
+		"${COMPUTED_ENV_VARS}", strings.Join(envVarLines, "\n"), // lacks beauty, but it works
+	), nil
+}