diff --git a/docs/adrs/flannel-options.md b/docs/adrs/flannel-options.md index 4cc3d00d09ab..ad20c28a5d58 100644 --- a/docs/adrs/flannel-options.md +++ b/docs/adrs/flannel-options.md @@ -47,7 +47,7 @@ K3s prepares both flannel flags, subnet, and backend configuration before starti | `--iface (string)`| Yes | `--flannel-iface` | | `--iface-can-reach (string)` | No** | N/A | | `--iface-regex (string)` | No** | N/A | -| `--ip-masq (bool)` | Yes/Hardcoded | ipv4 hardcoded to true. ipv6 configurable by k3s server flag `--flannel-ipv6-masq ` | +| `--ip-masq (bool)` | Yes | ipv4 and ipv6 configurable by k3s server flags `--flannel-ipv4-no-masq `/`--flannel-ipv6-masq ` | | `--iptables-forward-rules (bool)` | Hardcoded | True | | `--public-ip (string)` | Hardcoded | `--external-ip` when it is of IPv4 nature if k3s server flag `--flannel-external-ip` is true | | `--public-ipv6 (string)`| hardcoded | `--external-ip` when it is of IPv6 nature if k3s server flag `--flannel-external-ip` is true | @@ -73,7 +73,8 @@ To wrap up the context, k3s includes the following flannel options: | Server Flag | Type | Description | | --- | --- | --- | | `--flannel-backend` | string | Sets the encapsulation technology that will allow inter-node traffic to work. It matches part of the Backend parameter in the subnet&backend configuration | -| `--flannel-ipv6-masq` | bool | Enables masquerading traffic for IPv6. Note that IPv4 traffic is always masqueraded. It matches the ipv6 part of the `--ip-masq` flag | +| `--flannel-ipv4-no-masq` | bool | Disable masquerading traffic for IPv4. It matches the ipv4 part of the `--ip-masq` flag | +| `--flannel-ipv6-masq` | bool | Enables masquerading traffic for IPv6. It matches the ipv6 part of the `--ip-masq` flag | | `--flannel-external-ip` | bool | Enables using node external IP addresses for flannel traffic. It sort of matches on `--public-ip` and `--public-ipv6` flag. In this case, k3s selects the external-ips for those flags | ### Design suggestion @@ -95,6 +96,7 @@ Flannel-opt would have the following values: | Value | Old Flag | | --- | --- | +| `ipv4-masq` | `--flannel-ipv4-no-masq` | | `ipv6-masq` | `--flannel-ipv6-masq` | | `external-ip` | `--flannel-external-ip` | @@ -102,6 +104,7 @@ Similar to how we handle `--kubelet-arg`, both comma separated lists and repeate Examples of usage: `--flannel-opt=ipv6-masq,external-ip` (assumes true) +`--flannel-opt=ipv4-masq=true` `--flannel-opt=ipv6-masq=true` `--flannel-opt=ipv6-masq=true,external-ip=false` `--flannel-opt=ipv6-masq --flannel-opt=external-ip` diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 92b3c9d935bf..f5a7455eca32 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -614,6 +614,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N EnablePProf: envInfo.EnablePProf, EmbeddedRegistry: controlConfig.EmbeddedRegistry, FlannelBackend: controlConfig.FlannelBackend, + FlannelIPv4NoMasq: controlConfig.FlannelIPv4NoMasq, FlannelIPv6Masq: controlConfig.FlannelIPv6Masq, FlannelExternalIP: controlConfig.FlannelExternalIP, EgressSelectorMode: controlConfig.EgressSelectorMode, diff --git a/pkg/agent/flannel/flannel.go b/pkg/agent/flannel/flannel.go index 6b3824163f7c..c5758e9166d3 100644 --- a/pkg/agent/flannel/flannel.go +++ b/pkg/agent/flannel/flannel.go @@ -50,7 +50,7 @@ var ( FlannelExternalIPv6Annotation = FlannelBaseAnnotation + "/public-ipv6-overwrite" ) -func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string, flannelIPv6Masq bool, netMode int) error { +func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string, flannelIPv4NoMasq bool, flannelIPv6Masq bool, netMode int) error { extIface, err := LookupExtInterface(flannelIface, netMode) if err != nil { return errors.Wrap(err, "failed to find the interface") @@ -101,12 +101,21 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube prevIPv6Network := ReadIP6CIDRFromSubnetFile(subnetFile, "FLANNEL_IPV6_NETWORK") prevIPv6Subnet := ReadIP6CIDRFromSubnetFile(subnetFile, "FLANNEL_IPV6_SUBNET") + + masqNetwork := &config.Network + masqIPv6Network := &ip.IP6Net{} + + if flannelIPv4NoMasq { + //disable nat rules for IPv4 + masqNetwork = &ip.IP4Net{} + } + if flannelIPv6Masq { - err = trafficMngr.SetupAndEnsureMasqRules(ctx, config.Network, prevSubnet, prevNetwork, config.IPv6Network, prevIPv6Subnet, prevIPv6Network, bn.Lease(), 60) - } else { - //set empty flannel ipv6 Network to prevent masquerading - err = trafficMngr.SetupAndEnsureMasqRules(ctx, config.Network, prevSubnet, prevNetwork, ip.IP6Net{}, prevIPv6Subnet, prevIPv6Network, bn.Lease(), 60) + //enable nat rules for IPv6 + masqIPv6Network = &config.IPv6Network } + + err = trafficMngr.SetupAndEnsureMasqRules(ctx, *masqNetwork, prevSubnet, prevNetwork, *masqIPv6Network, prevIPv6Subnet, prevIPv6Network, bn.Lease(), 60) if err != nil { return errors.Wrap(err, "failed to setup masq rules") } @@ -114,6 +123,8 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube //setup forward rules trafficMngr.SetupAndEnsureForwardRules(ctx, config.Network, config.IPv6Network, 50) + //ipMask argument MUST stay hardcoded to true otherwise, the underlying bridge will apply MASQ rules + //for IPv4 and IPv6 as documented in https://github.com/flannel-io/cni-plugin/blob/main/README.md if err := WriteSubnetFile(subnetFile, config.Network, config.IPv6Network, true, bn, netMode); err != nil { // Continue, even though it failed. logrus.Warningf("Failed to write flannel subnet file: %s", err) @@ -264,7 +275,6 @@ func ReadCIDRsFromSubnetFile(path string, CIDRKey string) []ip.IP4Net { return prevCIDRs } - // ReadIP6CIDRFromSubnetFile reads the flannel subnet file and extracts the value of IPv6 network CIDRKey func ReadIP6CIDRFromSubnetFile(path string, CIDRKey string) ip.IP6Net { prevCIDRs := ReadIP6CIDRsFromSubnetFile(path, CIDRKey) diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index e25b45f5e5f7..1b5c03315e65 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -97,7 +97,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error { return errors.Wrap(err, "failed to check netMode for flannel") } go func() { - err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, kubeConfig, nodeConfig.FlannelIPv6Masq, netMode) + err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, kubeConfig, nodeConfig.FlannelIPv4NoMasq, nodeConfig.FlannelIPv6Masq, netMode) if err != nil && !errors.Is(err, context.Canceled) { logrus.Errorf("flannel exited: %v", err) os.Exit(1) diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index b30cb6ff324c..d32802bfe2d1 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -65,6 +65,7 @@ type Server struct { DisableScheduler bool ServerURL string FlannelBackend string + FlannelIPv4NoMasq bool FlannelIPv6Masq bool FlannelExternalIP bool EgressSelectorMode string @@ -242,6 +243,11 @@ var ServerFlags = []cli.Flag{ Destination: &ServerConfig.FlannelBackend, Value: "vxlan", }, + &cli.BoolFlag{ + Name: "flannel-ipv4-no-masq", + Usage: "(networking) Disable IPv4 masquerading for pod", + Destination: &ServerConfig.FlannelIPv4NoMasq, + }, &cli.BoolFlag{ Name: "flannel-ipv6-masq", Usage: "(networking) Enable IPv6 masquerading for pod", diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 4664caf6561e..a750e050118e 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -160,6 +160,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend + serverConfig.ControlConfig.FlannelIPv4NoMasq = cfg.FlannelIPv4NoMasq serverConfig.ControlConfig.FlannelIPv6Masq = cfg.FlannelIPv6Masq serverConfig.ControlConfig.FlannelExternalIP = cfg.FlannelExternalIP serverConfig.ControlConfig.EgressSelectorMode = cfg.EgressSelectorMode diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 9a74e95b5dbc..20e2efe0c161 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -49,6 +49,7 @@ type Node struct { FlannelConfFile string FlannelConfOverride bool FlannelIface *net.Interface + FlannelIPv4NoMasq bool FlannelIPv6Masq bool FlannelExternalIP bool EgressSelectorMode string @@ -180,6 +181,7 @@ type CriticalControlArgs struct { EncryptSecrets bool `cli:"secrets-encryption"` EmbeddedRegistry bool `cli:"embedded-registry"` FlannelBackend string `cli:"flannel-backend"` + FlannelIPv4NoMasq bool `cli:"flannel-ipv4-no-masq"` FlannelIPv6Masq bool `cli:"flannel-ipv6-masq"` FlannelExternalIP bool `cli:"flannel-external-ip"` EgressSelectorMode string `cli:"egress-selector-mode"`