diff --git a/README.md b/README.md index f5738899bd..b9620cfaff 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,19 @@ Currently, the only understood values for network Type are `OpenShiftSDN` and `O Other values are ignored. If you wish to use use a third-party network provider not managed by the operator, set the network type to something meaningful to you. The operator will not install or upgrade a network provider, but all other Network Operator functionality remains. +### Adding chained plugins +You can add raw CNI configuration snippets to be installed as CNI chained plugins. These will be included in the CNI configuration file used by the default network provider. Note that these are *not* multus multiple networks. Rather, they are for plugins that manipulate the container's existing network. For example, if you want to adjust your container's sysctls, you can configure + +```yaml +spec: + defaultNetwork: + type: OpenShiftSDN + openshiftSDNConfig: {} + chainedPlugins: + - '{"name": "tuning", "sysctl": { "net.core.somaxconn": "500"}}` +``` + +Not all default networks support chained plugins. Currently, only OpenShiftSDN does. ### Configuring OpenShiftSDN OpenShiftSDN supports the following configuration options, all of which are optional: @@ -116,14 +129,7 @@ These configuration flags are only in the Operator configuration object. Example: ```yaml -spec: - defaultNetwork: - type: OpenShiftSDN - openshiftSDNConfig: - mode: NetworkPolicy - vxlanPort: 4789 - mtu: 1450 - useExternalOpenvswitch: false + ``` ### Configuring OVNKubernetes diff --git a/bindata/network/openshift-sdn/sdn.yaml b/bindata/network/openshift-sdn/sdn.yaml index 892c7a101b..1e30210d0b 100644 --- a/bindata/network/openshift-sdn/sdn.yaml +++ b/bindata/network/openshift-sdn/sdn.yaml @@ -6,6 +6,9 @@ metadata: data: sdn-config.yaml: |- {{.NodeConfig | indent 4}} + 80-openshift.conflist: |- +{{.CNIConfig | indent 4}} + --- kind: DaemonSet apiVersion: apps/v1 @@ -48,7 +51,6 @@ spec: set -euo pipefail # if another process is listening on the cni-server socket, wait until it exits - trap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM retries=0 while true; do if echo 'test' | socat - UNIX-CONNECT:/var/run/openshift-sdn/cni-server.sock &>/dev/null; then @@ -64,24 +66,27 @@ spec: fi done - # local environment overrides + # Allow DEBUG_LOGLEVEL to be configured per-host. if [[ -f /etc/sysconfig/openshift-sdn ]]; then set -o allexport source /etc/sysconfig/openshift-sdn set +o allexport fi - #BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate - # Once this is released, then we can mount it properly + + {{/* BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; + clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate + Once this is released, then we can mount it properly */}} if [[ -d /etc/sysconfig/openshift-sdn ]]; then rmdir /etc/sysconfig/openshift-sdn || true fi - # Take over network functions on the node - rm -f /etc/cni/net.d/80-openshift-network.conf + # Remove current and old (v4.1) configuration files. + # The SDN process will write its CNI configuration file when the network is ready. + rm -f /etc/cni/net.d/80-openshift-network.conf {{.CNIPath}} cp -f /opt/cni/bin/* /host/opt/cni/bin/ # Launch the network process - exec /usr/bin/openshift-sdn --config=/config/sdn-config.yaml --url-only-kubeconfig=/etc/kubernetes/kubeconfig --loglevel=${DEBUG_LOGLEVEL:-2} + exec /usr/bin/openshift-sdn --config=/config/sdn-config.yaml --url-only-kubeconfig=/etc/kubernetes/kubeconfig --loglevel=${DEBUG_LOGLEVEL:-2} --cni-config-file-in=/config/80-openshift.conflist --cni-config-file-out={{.CNIPath}} securityContext: privileged: true volumeMounts: @@ -142,7 +147,7 @@ spec: lifecycle: preStop: exec: - command: ["rm","-f","/etc/cni/net.d/80-openshift-network.conf", "/host/opt/cni/bin/openshift-sdn"] + command: ["rm","-f","{{.CNIPath}}", "/host/opt/cni/bin/openshift-sdn"] # this comes from the kube-proxy code livenessProbe: httpGet: @@ -151,7 +156,7 @@ spec: readinessProbe: exec: # openshift-sdn writes this file when it is ready to handle pod requests. - command: ["test", "-f", "/etc/cni/net.d/80-openshift-network.conf"] + command: ["test", "-f", "{{.CNIPath}}"] initialDelaySeconds: 5 periodSeconds: 5 nodeSelector: diff --git a/pkg/network/cni.go b/pkg/network/cni.go new file mode 100644 index 0000000000..d0afc9f598 --- /dev/null +++ b/pkg/network/cni.go @@ -0,0 +1,66 @@ +package network + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/pkg/errors" + + operv1 "github.com/openshift/api/operator/v1" +) + +const CNINetworkName = "openshift" + +// Whatever network is the default network should use this as the target CNI +// configuration file. +const CNIConfigPath = "/etc/cni/net.d/80-openshift.conflist" + +// validateChainedPlugins ensures that all supplied chained plugins are some +// kind of reasonable cni configuration +func validateChainedPlugins(conf *operv1.NetworkSpec) []error { + out := []error{} + + if len(conf.DefaultNetwork.ChainedPlugins) > 0 && conf.DefaultNetwork.Type != operv1.NetworkTypeOpenShiftSDN { + out = append(out, errors.Errorf("network type %s does not support chained plugins", conf.DefaultNetwork.Type)) + } + + for i, entry := range conf.DefaultNetwork.ChainedPlugins { + if entry.RawCNIConfig == "" { + out = append(out, errors.Errorf("invalid CNI plugin entry Spec.DefaultNetwork.ChainedPlugin[%d]: rawCNIConfig must be specified", i)) + continue + } + var m map[string]interface{} + if err := json.Unmarshal([]byte(entry.RawCNIConfig), &m); err != nil { + out = append(out, errors.Wrapf(err, "invalid json in Spec.DefaultNetwork.ChainedPlugin[%d].RawCNIConfig", i)) + continue + } + + if _, ok := m["type"]; !ok { + out = append(out, errors.Errorf("invalid CNI plugin entry in Spec.DefaultNetwork.ChainedPlugin[%d].RawCNIConfig: must have 'type' key", i)) + } + } + + return out +} + +// makeCNIConfig merges the primary plugin configuration (as JSON) with any chained +// plugin's configurations. +func makeCNIConfig(conf *operv1.NetworkSpec, netName, cniVersion, primaryPluginConfig string) string { + pluginConfigs := []string{primaryPluginConfig} + for _, entry := range conf.DefaultNetwork.ChainedPlugins { + pluginConfigs = append(pluginConfigs, entry.RawCNIConfig) + } + + configFile := fmt.Sprintf(` +{ + "name": "%s", + "cniVersion": "%s", + "plugins": [ + %s + ] +}`, + netName, cniVersion, strings.Join(pluginConfigs, ",\n\t")) + + return configFile +} diff --git a/pkg/network/cni_test.go b/pkg/network/cni_test.go new file mode 100644 index 0000000000..d4d2ec079f --- /dev/null +++ b/pkg/network/cni_test.go @@ -0,0 +1,77 @@ +package network + +import ( + "testing" + + operv1 "github.com/openshift/api/operator/v1" + + . "github.com/onsi/gomega" +) + +func TestMakeCNIConfig(t *testing.T) { + g := NewGomegaWithT(t) + conf := operv1.NetworkSpec{} + + out := makeCNIConfig(&conf, "test1", "0.1.2", `{"type": "only"}`) + g.Expect(out).To(MatchJSON(` +{ + "cniVersion": "0.1.2", + "name": "test1", + "plugins": [{"type": "only"}] +}`)) + + conf.DefaultNetwork.ChainedPlugins = []operv1.ChainedPluginEntry{ + {RawCNIConfig: `{"type": "foo"}`}, + {RawCNIConfig: `{"type": "bar", "a": "b", "c":{"a": "b", "c": "d"}}`}, + } + + out = makeCNIConfig(&conf, "test2", "1.2.3", `{"type": "primary"}`) + g.Expect(out).To(MatchJSON(` +{ + "cniVersion": "1.2.3", + "name": "test2", + "plugins": [ + {"type": "primary"}, + {"type": "foo"}, + {"type": "bar", + "a": "b", + "c": { + "a": "b", + "c": "d" + } + }] +}`)) + +} + +func TestValidateChainedPlugins(t *testing.T) { + g := NewGomegaWithT(t) + + conf := &operv1.NetworkSpec{} + g.Expect(validateChainedPlugins(conf)).To(BeEmpty()) + + conf.DefaultNetwork.ChainedPlugins = []operv1.ChainedPluginEntry{ + {RawCNIConfig: `{"type": "foo"}`}, + {RawCNIConfig: `{"type": "bar"}`}, + } + conf.DefaultNetwork.Type = "unknown" + g.Expect(validateChainedPlugins(conf)).To(ContainElement(MatchError( + ContainSubstring("network type unknown does not support chained plugins")))) + + conf.DefaultNetwork.Type = "OpenShiftSDN" + g.Expect(validateChainedPlugins(conf)).To(BeEmpty()) + + conf.DefaultNetwork.ChainedPlugins = []operv1.ChainedPluginEntry{ + {RawCNIConfig: `asdfasdf`}, + } + g.Expect(validateChainedPlugins(conf)).To(ContainElement(MatchError( + ContainSubstring("invalid json in Spec.DefaultNetwork.ChainedPlugin[0].RawCNIConfig")))) + + conf.DefaultNetwork.ChainedPlugins = []operv1.ChainedPluginEntry{ + {RawCNIConfig: `{"type": "foo"}`}, + {RawCNIConfig: `{"name": "bar"}`}, + } + g.Expect(validateChainedPlugins(conf)).To(ContainElement(MatchError( + ContainSubstring("invalid CNI plugin entry in Spec.DefaultNetwork.ChainedPlugin[1].RawCNIConfig: must have 'type' key")))) + +} diff --git a/pkg/network/openshift_sdn.go b/pkg/network/openshift_sdn.go index 8aaa565871..6c967a0474 100644 --- a/pkg/network/openshift_sdn.go +++ b/pkg/network/openshift_sdn.go @@ -39,6 +39,7 @@ func renderOpenShiftSDN(conf *operv1.NetworkSpec, manifestDir string) ([]*uns.Un data.Data["KUBERNETES_SERVICE_HOST"] = os.Getenv("KUBERNETES_SERVICE_HOST") data.Data["KUBERNETES_SERVICE_PORT"] = os.Getenv("KUBERNETES_SERVICE_PORT") data.Data["Mode"] = c.Mode + data.Data["CNIPath"] = CNIConfigPath operCfg, err := controllerConfig(conf) if err != nil { @@ -52,6 +53,9 @@ func renderOpenShiftSDN(conf *operv1.NetworkSpec, manifestDir string) ([]*uns.Un } data.Data["NodeConfig"] = nodeCfg + data.Data["CNIConfig"] = makeCNIConfig(conf, CNINetworkName, "0.3.1", + `{"type": "openshift-sdn"}`) + manifests, err := render.RenderDir(filepath.Join(manifestDir, "network/openshift-sdn"), &data) if err != nil { return nil, errors.Wrap(err, "failed to render manifests") @@ -105,7 +109,7 @@ func isOpenShiftSDNChangeSafe(prev, next *operv1.NetworkSpec) []error { } func fillOpenShiftSDNDefaults(conf, previous *operv1.NetworkSpec, hostMTU int) { - // NOTE: If you change any defaults, and it's not a safe chang to roll out + // NOTE: If you change any defaults, and it's not a safe change to roll out // to existing clusters, you MUST use the value from previous instead. if conf.DeployKubeProxy == nil { prox := false diff --git a/pkg/network/openshift_sdn_test.go b/pkg/network/openshift_sdn_test.go index bb7684b7f2..8ebb0c96a7 100644 --- a/pkg/network/openshift_sdn_test.go +++ b/pkg/network/openshift_sdn_test.go @@ -434,3 +434,60 @@ volumeConfig: volumeDirectory: "" `)) } + +func TestCNIConfig(t *testing.T) { + g := NewGomegaWithT(t) + + crd := OpenShiftSDNConfig.DeepCopy() + config := &crd.Spec + FillDefaults(config, nil) + + // iter through all objects, finding the sdn config map + getCNIConfigFile := func(objs []*uns.Unstructured) string { + for _, obj := range objs { + if obj.GetKind() == "ConfigMap" && obj.GetName() == "sdn-config" { + val, ok, err := uns.NestedString(obj.Object, "data", "80-openshift.conflist") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ok).To(BeTrue()) + return val + + } + } + t.Fatal("failed to find sdn-config") + return "" //unreachable + } + + // test default rendering + objs, err := renderOpenShiftSDN(config, manifestDir) + g.Expect(err).NotTo(HaveOccurred()) + cfg := getCNIConfigFile(objs) + + g.Expect(cfg).To(MatchJSON(` +{ + "name": "openshift", + "cniVersion": "0.3.1", + "plugins": [ + { "type": "openshift-sdn" } + ] +}`)) + + config.DefaultNetwork.ChainedPlugins = []operv1.ChainedPluginEntry{ + {RawCNIConfig: `{"type": "foo"}`}, + {RawCNIConfig: `{"type": "bar"}`}, + } + + objs, err = renderOpenShiftSDN(config, manifestDir) + g.Expect(err).NotTo(HaveOccurred()) + cfg = getCNIConfigFile(objs) + + g.Expect(cfg).To(MatchJSON(` +{ + "name": "openshift", + "cniVersion": "0.3.1", + "plugins": [ + { "type": "openshift-sdn" }, + { "type": "foo" }, + { "type": "bar" } + ] +}`)) +} diff --git a/pkg/network/render.go b/pkg/network/render.go index 1d8eb89d03..5e00971e97 100644 --- a/pkg/network/render.go +++ b/pkg/network/render.go @@ -197,13 +197,15 @@ func ValidateMultus(conf *operv1.NetworkSpec) []error { // ValidateDefaultNetwork validates whichever network is specified // as the default network. func ValidateDefaultNetwork(conf *operv1.NetworkSpec) []error { + out := validateChainedPlugins(conf) + switch conf.DefaultNetwork.Type { case operv1.NetworkTypeOpenShiftSDN: - return validateOpenShiftSDN(conf) + return append(out, validateOpenShiftSDN(conf)...) case operv1.NetworkTypeOVNKubernetes: - return validateOVNKubernetes(conf) + return append(out, validateOVNKubernetes(conf)...) default: - return nil + return out } } @@ -242,6 +244,8 @@ func IsDefaultNetworkChangeSafe(prev, next *operv1.NetworkSpec) []error { return []error{errors.Errorf("cannot change default network type")} } + // It is allowed to change Spec.DefaultNetwork.ChainedPlugins + switch prev.DefaultNetwork.Type { case operv1.NetworkTypeOpenShiftSDN: return isOpenShiftSDNChangeSafe(prev, next) diff --git a/vendor/github.com/openshift/api/operator/v1/types_network.go b/vendor/github.com/openshift/api/operator/v1/types_network.go index 8ba638ce9d..cc8dc97cf1 100644 --- a/vendor/github.com/openshift/api/operator/v1/types_network.go +++ b/vendor/github.com/openshift/api/operator/v1/types_network.go @@ -94,6 +94,19 @@ type DefaultNetworkDefinition struct { // not implemented. // +optional OVNKubernetesConfig *OVNKubernetesConfig `json:"ovnKubernetesConfig,omitempty"` + + // chainedPlugins is an array of CNI plugins to append as chained plugins + // as part of the default network CNI configuration. + // +optional + ChainedPlugins []ChainedPluginEntry `json:"chainedPlugins,omitempty"` +} + +// ChainedPluginEntry is a single chained plugin to install. +type ChainedPluginEntry struct { + // rawCNIConfig is a raw CNI json snippet to add to the "plugins" array. + // If you use this, you must ensure that the referenced CNI binary is already + // written to disk. + RawCNIConfig string `json:"rawCNIConfig,omitempty"` } // AdditionalNetworkDefinition configures an extra network that is available but not