Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,19 @@ Currently, the only understood values for network Type are `OpenShiftSDN` and `O

Other values are ignored. If you wish to use use a third-party network provider not managed by the operator, set the network type to something meaningful to you. The operator will not install or upgrade a network provider, but all other Network Operator functionality remains.

### Adding chained plugins
You can add raw CNI configuration snippets to be installed as CNI chained plugins. These will be included in the CNI configuration file used by the default network provider. Note that these are *not* multus multiple networks. Rather, they are for plugins that manipulate the container's existing network. For example, if you want to adjust your container's sysctls, you can configure
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should note that this applies to all containers in the cluster.


```yaml
spec:
defaultNetwork:
type: OpenShiftSDN
openshiftSDNConfig: {}
chainedPlugins:
- '{"name": "tuning", "sysctl": { "net.core.somaxconn": "500"}}`
```

Not all default networks support chained plugins. Currently, only OpenShiftSDN does.

### Configuring OpenShiftSDN
OpenShiftSDN supports the following configuration options, all of which are optional:
Expand All @@ -116,14 +129,7 @@ These configuration flags are only in the Operator configuration object.

Example:
```yaml
spec:
defaultNetwork:
type: OpenShiftSDN
openshiftSDNConfig:
mode: NetworkPolicy
vxlanPort: 4789
mtu: 1450
useExternalOpenvswitch: false

```

### Configuring OVNKubernetes
Expand Down
23 changes: 14 additions & 9 deletions bindata/network/openshift-sdn/sdn.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ metadata:
data:
sdn-config.yaml: |-
{{.NodeConfig | indent 4}}
80-openshift.conflist: |-
{{.CNIConfig | indent 4}}

---
kind: DaemonSet
apiVersion: apps/v1
Expand Down Expand Up @@ -48,7 +51,6 @@ spec:
set -euo pipefail

# if another process is listening on the cni-server socket, wait until it exits
trap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM
retries=0
while true; do
if echo 'test' | socat - UNIX-CONNECT:/var/run/openshift-sdn/cni-server.sock &>/dev/null; then
Expand All @@ -64,24 +66,27 @@ spec:
fi
done

# local environment overrides
# Allow DEBUG_LOGLEVEL to be configured per-host.
if [[ -f /etc/sysconfig/openshift-sdn ]]; then
set -o allexport
source /etc/sysconfig/openshift-sdn
set +o allexport
fi
#BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate; clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate
# Once this is released, then we can mount it properly

{{/* BUG: cdc accidentally mounted /etc/sysconfig/openshift-sdn as DirectoryOrCreate;
clean it up so we can ultimately mount /etc/sysconfig/openshift-sdn as FileOrCreate
Once this is released, then we can mount it properly */}}
if [[ -d /etc/sysconfig/openshift-sdn ]]; then
rmdir /etc/sysconfig/openshift-sdn || true
fi

# Take over network functions on the node
rm -f /etc/cni/net.d/80-openshift-network.conf
# Remove current and old (v4.1) configuration files.
# The SDN process will write its CNI configuration file when the network is ready.
rm -f /etc/cni/net.d/80-openshift-network.conf {{.CNIPath}}
cp -f /opt/cni/bin/* /host/opt/cni/bin/

# Launch the network process
exec /usr/bin/openshift-sdn --config=/config/sdn-config.yaml --url-only-kubeconfig=/etc/kubernetes/kubeconfig --loglevel=${DEBUG_LOGLEVEL:-2}
exec /usr/bin/openshift-sdn --config=/config/sdn-config.yaml --url-only-kubeconfig=/etc/kubernetes/kubeconfig --loglevel=${DEBUG_LOGLEVEL:-2} --cni-config-file-in=/config/80-openshift.conflist --cni-config-file-out={{.CNIPath}}
securityContext:
privileged: true
volumeMounts:
Expand Down Expand Up @@ -142,7 +147,7 @@ spec:
lifecycle:
preStop:
exec:
command: ["rm","-f","/etc/cni/net.d/80-openshift-network.conf", "/host/opt/cni/bin/openshift-sdn"]
command: ["rm","-f","{{.CNIPath}}", "/host/opt/cni/bin/openshift-sdn"]
# this comes from the kube-proxy code
livenessProbe:
httpGet:
Expand All @@ -151,7 +156,7 @@ spec:
readinessProbe:
exec:
# openshift-sdn writes this file when it is ready to handle pod requests.
command: ["test", "-f", "/etc/cni/net.d/80-openshift-network.conf"]
command: ["test", "-f", "{{.CNIPath}}"]
initialDelaySeconds: 5
periodSeconds: 5
nodeSelector:
Expand Down
66 changes: 66 additions & 0 deletions pkg/network/cni.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package network

import (
"encoding/json"
"fmt"
"strings"

"github.com/pkg/errors"

operv1 "github.com/openshift/api/operator/v1"
)

const CNINetworkName = "openshift"

// Whatever network is the default network should use this as the target CNI
// configuration file.
const CNIConfigPath = "/etc/cni/net.d/80-openshift.conflist"

// validateChainedPlugins ensures that all supplied chained plugins are some
// kind of reasonable cni configuration
func validateChainedPlugins(conf *operv1.NetworkSpec) []error {
out := []error{}

if len(conf.DefaultNetwork.ChainedPlugins) > 0 && conf.DefaultNetwork.Type != operv1.NetworkTypeOpenShiftSDN {
out = append(out, errors.Errorf("network type %s does not support chained plugins", conf.DefaultNetwork.Type))
}

for i, entry := range conf.DefaultNetwork.ChainedPlugins {
if entry.RawCNIConfig == "" {
out = append(out, errors.Errorf("invalid CNI plugin entry Spec.DefaultNetwork.ChainedPlugin[%d]: rawCNIConfig must be specified", i))
continue
}
var m map[string]interface{}
if err := json.Unmarshal([]byte(entry.RawCNIConfig), &m); err != nil {
out = append(out, errors.Wrapf(err, "invalid json in Spec.DefaultNetwork.ChainedPlugin[%d].RawCNIConfig", i))
continue
}

if _, ok := m["type"]; !ok {
out = append(out, errors.Errorf("invalid CNI plugin entry in Spec.DefaultNetwork.ChainedPlugin[%d].RawCNIConfig: must have 'type' key", i))
}
}

return out
}

// makeCNIConfig merges the primary plugin configuration (as JSON) with any chained
// plugin's configurations.
func makeCNIConfig(conf *operv1.NetworkSpec, netName, cniVersion, primaryPluginConfig string) string {
pluginConfigs := []string{primaryPluginConfig}
for _, entry := range conf.DefaultNetwork.ChainedPlugins {
pluginConfigs = append(pluginConfigs, entry.RawCNIConfig)
}

configFile := fmt.Sprintf(`
{
"name": "%s",
"cniVersion": "%s",
"plugins": [
%s
]
}`,
netName, cniVersion, strings.Join(pluginConfigs, ",\n\t"))

return configFile
}
77 changes: 77 additions & 0 deletions pkg/network/cni_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package network

import (
"testing"

operv1 "github.com/openshift/api/operator/v1"

. "github.com/onsi/gomega"
)

func TestMakeCNIConfig(t *testing.T) {
g := NewGomegaWithT(t)
conf := operv1.NetworkSpec{}

out := makeCNIConfig(&conf, "test1", "0.1.2", `{"type": "only"}`)
g.Expect(out).To(MatchJSON(`
{
"cniVersion": "0.1.2",
"name": "test1",
"plugins": [{"type": "only"}]
}`))

conf.DefaultNetwork.ChainedPlugins = []operv1.ChainedPluginEntry{
{RawCNIConfig: `{"type": "foo"}`},
{RawCNIConfig: `{"type": "bar", "a": "b", "c":{"a": "b", "c": "d"}}`},
}

out = makeCNIConfig(&conf, "test2", "1.2.3", `{"type": "primary"}`)
g.Expect(out).To(MatchJSON(`
{
"cniVersion": "1.2.3",
"name": "test2",
"plugins": [
{"type": "primary"},
{"type": "foo"},
{"type": "bar",
"a": "b",
"c": {
"a": "b",
"c": "d"
}
}]
}`))

}

func TestValidateChainedPlugins(t *testing.T) {
g := NewGomegaWithT(t)

conf := &operv1.NetworkSpec{}
g.Expect(validateChainedPlugins(conf)).To(BeEmpty())

conf.DefaultNetwork.ChainedPlugins = []operv1.ChainedPluginEntry{
{RawCNIConfig: `{"type": "foo"}`},
{RawCNIConfig: `{"type": "bar"}`},
}
conf.DefaultNetwork.Type = "unknown"
g.Expect(validateChainedPlugins(conf)).To(ContainElement(MatchError(
ContainSubstring("network type unknown does not support chained plugins"))))

conf.DefaultNetwork.Type = "OpenShiftSDN"
g.Expect(validateChainedPlugins(conf)).To(BeEmpty())

conf.DefaultNetwork.ChainedPlugins = []operv1.ChainedPluginEntry{
{RawCNIConfig: `asdfasdf`},
}
g.Expect(validateChainedPlugins(conf)).To(ContainElement(MatchError(
ContainSubstring("invalid json in Spec.DefaultNetwork.ChainedPlugin[0].RawCNIConfig"))))

conf.DefaultNetwork.ChainedPlugins = []operv1.ChainedPluginEntry{
{RawCNIConfig: `{"type": "foo"}`},
{RawCNIConfig: `{"name": "bar"}`},
}
g.Expect(validateChainedPlugins(conf)).To(ContainElement(MatchError(
ContainSubstring("invalid CNI plugin entry in Spec.DefaultNetwork.ChainedPlugin[1].RawCNIConfig: must have 'type' key"))))

}
6 changes: 5 additions & 1 deletion pkg/network/openshift_sdn.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ func renderOpenShiftSDN(conf *operv1.NetworkSpec, manifestDir string) ([]*uns.Un
data.Data["KUBERNETES_SERVICE_HOST"] = os.Getenv("KUBERNETES_SERVICE_HOST")
data.Data["KUBERNETES_SERVICE_PORT"] = os.Getenv("KUBERNETES_SERVICE_PORT")
data.Data["Mode"] = c.Mode
data.Data["CNIPath"] = CNIConfigPath

operCfg, err := controllerConfig(conf)
if err != nil {
Expand All @@ -52,6 +53,9 @@ func renderOpenShiftSDN(conf *operv1.NetworkSpec, manifestDir string) ([]*uns.Un
}
data.Data["NodeConfig"] = nodeCfg

data.Data["CNIConfig"] = makeCNIConfig(conf, CNINetworkName, "0.3.1",
`{"type": "openshift-sdn"}`)
Copy link
Contributor

@dcbw dcbw May 28, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not super happy about this, can we just have the SDN itself do the combination? eg the CNO makes the template from the ChainedPlugins and the SDN inserts itself at position 0 in the list and then writes the result conflist to the final CNI configdir that CRIO looks for?

eg I'd rather have the SDN plugin itself handle its config rather than encode logic about the conf/type/version/etc in the operator...

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was going to say something similar before, but there is already a crazy amount of coupling between the operator and the SDN plugin itself anyway. This doesn't really make it worse. (And besides, what happens if one of the chained plugins wants to use some feature that requires using version 0.4.0 of the spec?)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I thought about that for a long time, and couldn't find a solution I like.

First of all, the cniVersion is a tricky coordination point no matter where you place it. It needs to be the same for all plugins, and all plugins need to support the selected versions. If we bump openshift-sdn to 0.4.0, then all chained plugins instantly must support 0.4.0. I suppose this is a problem with CNI, and we may wish to revisit that decision. Wherever we do the config munging doesn't fix this problem.

As for having openshift-sdn insert itself, I decided against that for a few reasons:

  1. In general, manipulating untyped json is a pain in go :-)
  2. This is easier add to other plugins, e.g. ovn. We can just write a sidecar in bash that waits for healthz and copies a file.
  3. Generally, CNI configuration is expected to be administrator-defined. openshift-sdn writes it's own configuration file for availability signalling and historical reasons. And because it has no knobs.

I'm happy to revisit this. It's not great either way you look at it.


manifests, err := render.RenderDir(filepath.Join(manifestDir, "network/openshift-sdn"), &data)
if err != nil {
return nil, errors.Wrap(err, "failed to render manifests")
Expand Down Expand Up @@ -105,7 +109,7 @@ func isOpenShiftSDNChangeSafe(prev, next *operv1.NetworkSpec) []error {
}

func fillOpenShiftSDNDefaults(conf, previous *operv1.NetworkSpec, hostMTU int) {
// NOTE: If you change any defaults, and it's not a safe chang to roll out
// NOTE: If you change any defaults, and it's not a safe change to roll out
// to existing clusters, you MUST use the value from previous instead.
if conf.DeployKubeProxy == nil {
prox := false
Expand Down
57 changes: 57 additions & 0 deletions pkg/network/openshift_sdn_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -434,3 +434,60 @@ volumeConfig:
volumeDirectory: ""
`))
}

func TestCNIConfig(t *testing.T) {
g := NewGomegaWithT(t)

crd := OpenShiftSDNConfig.DeepCopy()
config := &crd.Spec
FillDefaults(config, nil)

// iter through all objects, finding the sdn config map
getCNIConfigFile := func(objs []*uns.Unstructured) string {
for _, obj := range objs {
if obj.GetKind() == "ConfigMap" && obj.GetName() == "sdn-config" {
val, ok, err := uns.NestedString(obj.Object, "data", "80-openshift.conflist")
g.Expect(err).NotTo(HaveOccurred())
g.Expect(ok).To(BeTrue())
return val

}
}
t.Fatal("failed to find sdn-config")
return "" //unreachable
}

// test default rendering
objs, err := renderOpenShiftSDN(config, manifestDir)
g.Expect(err).NotTo(HaveOccurred())
cfg := getCNIConfigFile(objs)

g.Expect(cfg).To(MatchJSON(`
{
"name": "openshift",
"cniVersion": "0.3.1",
"plugins": [
{ "type": "openshift-sdn" }
]
}`))

config.DefaultNetwork.ChainedPlugins = []operv1.ChainedPluginEntry{
{RawCNIConfig: `{"type": "foo"}`},
{RawCNIConfig: `{"type": "bar"}`},
}

objs, err = renderOpenShiftSDN(config, manifestDir)
g.Expect(err).NotTo(HaveOccurred())
cfg = getCNIConfigFile(objs)

g.Expect(cfg).To(MatchJSON(`
{
"name": "openshift",
"cniVersion": "0.3.1",
"plugins": [
{ "type": "openshift-sdn" },
{ "type": "foo" },
{ "type": "bar" }
]
}`))
}
10 changes: 7 additions & 3 deletions pkg/network/render.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,13 +197,15 @@ func ValidateMultus(conf *operv1.NetworkSpec) []error {
// ValidateDefaultNetwork validates whichever network is specified
// as the default network.
func ValidateDefaultNetwork(conf *operv1.NetworkSpec) []error {
out := validateChainedPlugins(conf)

switch conf.DefaultNetwork.Type {
case operv1.NetworkTypeOpenShiftSDN:
return validateOpenShiftSDN(conf)
return append(out, validateOpenShiftSDN(conf)...)
case operv1.NetworkTypeOVNKubernetes:
return validateOVNKubernetes(conf)
return append(out, validateOVNKubernetes(conf)...)
default:
return nil
return out
}
}

Expand Down Expand Up @@ -242,6 +244,8 @@ func IsDefaultNetworkChangeSafe(prev, next *operv1.NetworkSpec) []error {
return []error{errors.Errorf("cannot change default network type")}
}

// It is allowed to change Spec.DefaultNetwork.ChainedPlugins

switch prev.DefaultNetwork.Type {
case operv1.NetworkTypeOpenShiftSDN:
return isOpenShiftSDNChangeSafe(prev, next)
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.