Ipv4 no masq

docs/adrs/flannel-options.md

@@ -47,7 +47,7 @@ K3s prepares both flannel flags, subnet, and backend configuration before starti
 | `--iface (string)`| Yes | `--flannel-iface` |
 | `--iface-can-reach (string)` | No** | N/A |
 | `--iface-regex (string)` | No** | N/A |
-| `--ip-masq (bool)` | Yes/Hardcoded | ipv4 hardcoded to true. ipv6 configurable by k3s server flag `--flannel-ipv6-masq ` |
+| `--ip-masq (bool)` | Yes | ipv4 and ipv6 configurable by k3s server flags `--flannel-ipv4-no-masq `/`--flannel-ipv6-masq ` |
 | `--iptables-forward-rules (bool)` | Hardcoded | True |
 | `--public-ip (string)` | Hardcoded | `--external-ip` when it is of IPv4 nature if k3s server flag `--flannel-external-ip` is true |
 | `--public-ipv6 (string)`| hardcoded | `--external-ip` when it is of IPv6 nature if k3s server flag `--flannel-external-ip` is true |
@@ -73,7 +73,8 @@ To wrap up the context, k3s includes the following flannel options:
 | Server Flag | Type | Description |
 | --- | --- | --- |
 | `--flannel-backend` | string | Sets the encapsulation technology that will allow inter-node traffic to work. It matches part of the Backend parameter in the subnet&backend configuration |
-| `--flannel-ipv6-masq` | bool | Enables masquerading traffic for IPv6. Note that IPv4 traffic is always masqueraded. It matches the ipv6 part of the `--ip-masq` flag |
+| `--flannel-ipv4-no-masq` | bool | Disable masquerading traffic for IPv4. It matches the ipv4 part of the `--ip-masq` flag |
+| `--flannel-ipv6-masq` | bool | Enables masquerading traffic for IPv6. It matches the ipv6 part of the `--ip-masq` flag |
 | `--flannel-external-ip` | bool | Enables using node external IP addresses for flannel traffic. It sort of matches on `--public-ip` and `--public-ipv6` flag. In this case, k3s selects the external-ips for those flags |
 
 ### Design suggestion
@@ -95,13 +96,15 @@ Flannel-opt would have the following values:
 
 | Value | Old Flag |
 | --- | --- |
+| `ipv4-masq` | `--flannel-ipv4-no-masq` |
 | `ipv6-masq` | `--flannel-ipv6-masq` |
 | `external-ip` | `--flannel-external-ip` |
 
 Similar to how we handle `--kubelet-arg`, both comma separated lists and repeated args would be accepted.
 
 Examples of usage:  
 `--flannel-opt=ipv6-masq,external-ip` (assumes true)  
+`--flannel-opt=ipv4-masq=true`  
 `--flannel-opt=ipv6-masq=true`  
 `--flannel-opt=ipv6-masq=true,external-ip=false`  
 `--flannel-opt=ipv6-masq --flannel-opt=external-ip`

pkg/agent/config/config.go

@@ -614,6 +614,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		EnablePProf:              envInfo.EnablePProf,
 		EmbeddedRegistry:         controlConfig.EmbeddedRegistry,
 		FlannelBackend:           controlConfig.FlannelBackend,
+		FlannelIPv4NoMasq:        controlConfig.FlannelIPv4NoMasq,
 		FlannelIPv6Masq:          controlConfig.FlannelIPv6Masq,
 		FlannelExternalIP:        controlConfig.FlannelExternalIP,
 		EgressSelectorMode:       controlConfig.EgressSelectorMode,

pkg/agent/flannel/flannel.go

@@ -50,7 +50,7 @@ var (
 	FlannelExternalIPv6Annotation = FlannelBaseAnnotation + "/public-ipv6-overwrite"
 )
 
-func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string, flannelIPv6Masq bool, netMode int) error {
+func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string, flannelIPv4NoMasq bool, flannelIPv6Masq bool, netMode int) error {
 	extIface, err := LookupExtInterface(flannelIface, netMode)
 	if err != nil {
 		return errors.Wrap(err, "failed to find the interface")
@@ -101,19 +101,30 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 
 	prevIPv6Network := ReadIP6CIDRFromSubnetFile(subnetFile, "FLANNEL_IPV6_NETWORK")
 	prevIPv6Subnet := ReadIP6CIDRFromSubnetFile(subnetFile, "FLANNEL_IPV6_SUBNET")
+
+	masqNetwork := &config.Network
+	masqIPv6Network := &ip.IP6Net{}
+
+	if flannelIPv4NoMasq {
+		//disable nat rules for IPv4
+		masqNetwork = &ip.IP4Net{}
+	}
+
 	if flannelIPv6Masq {
-		err = trafficMngr.SetupAndEnsureMasqRules(ctx, config.Network, prevSubnet, prevNetwork, config.IPv6Network, prevIPv6Subnet, prevIPv6Network, bn.Lease(), 60)
-	} else {
-		//set empty flannel ipv6 Network to prevent masquerading
-		err = trafficMngr.SetupAndEnsureMasqRules(ctx, config.Network, prevSubnet, prevNetwork, ip.IP6Net{}, prevIPv6Subnet, prevIPv6Network, bn.Lease(), 60)
+		//enable nat rules for IPv6
+		masqIPv6Network = &config.IPv6Network
 	}
+
+	err = trafficMngr.SetupAndEnsureMasqRules(ctx, *masqNetwork, prevSubnet, prevNetwork, *masqIPv6Network, prevIPv6Subnet, prevIPv6Network, bn.Lease(), 60)
 	if err != nil {
 		return errors.Wrap(err, "failed to setup masq rules")
 	}
 
 	//setup forward rules
 	trafficMngr.SetupAndEnsureForwardRules(ctx, config.Network, config.IPv6Network, 50)
 
+	//ipMask argument MUST stay hardcoded to true otherwise, the underlying bridge will apply MASQ rules
+	//for IPv4 and IPv6 as documented in https://github.com/flannel-io/cni-plugin/blob/main/README.md
 	if err := WriteSubnetFile(subnetFile, config.Network, config.IPv6Network, true, bn, netMode); err != nil {
 		// Continue, even though it failed.
 		logrus.Warningf("Failed to write flannel subnet file: %s", err)
@@ -264,7 +275,6 @@ func ReadCIDRsFromSubnetFile(path string, CIDRKey string) []ip.IP4Net {
 	return prevCIDRs
 }
 
-
 // ReadIP6CIDRFromSubnetFile reads the flannel subnet file and extracts the value of IPv6 network CIDRKey
 func ReadIP6CIDRFromSubnetFile(path string, CIDRKey string) ip.IP6Net {
 	prevCIDRs := ReadIP6CIDRsFromSubnetFile(path, CIDRKey)

pkg/agent/flannel/setup.go

@@ -97,7 +97,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 		return errors.Wrap(err, "failed to check netMode for flannel")
 	}
 	go func() {
-		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, kubeConfig, nodeConfig.FlannelIPv6Masq, netMode)
+		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, kubeConfig, nodeConfig.FlannelIPv4NoMasq, nodeConfig.FlannelIPv6Masq, netMode)
 		if err != nil && !errors.Is(err, context.Canceled) {
 			logrus.Errorf("flannel exited: %v", err)
 			os.Exit(1)

pkg/cli/cmds/server.go

@@ -65,6 +65,7 @@ type Server struct {
 	DisableScheduler         bool
 	ServerURL                string
 	FlannelBackend           string
+	FlannelIPv4NoMasq        bool
 	FlannelIPv6Masq          bool
 	FlannelExternalIP        bool
 	EgressSelectorMode       string
@@ -242,6 +243,11 @@ var ServerFlags = []cli.Flag{
 		Destination: &ServerConfig.FlannelBackend,
 		Value:       "vxlan",
 	},
+	&cli.BoolFlag{
+		Name:        "flannel-ipv4-no-masq",
+		Usage:       "(networking) Disable IPv4 masquerading for pod",
+		Destination: &ServerConfig.FlannelIPv4NoMasq,
+	},
 	&cli.BoolFlag{
 		Name:        "flannel-ipv6-masq",
 		Usage:       "(networking) Enable IPv6 masquerading for pod",

pkg/cli/server/server.go

@@ -160,6 +160,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
 	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
 	serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
+	serverConfig.ControlConfig.FlannelIPv4NoMasq = cfg.FlannelIPv4NoMasq
 	serverConfig.ControlConfig.FlannelIPv6Masq = cfg.FlannelIPv6Masq
 	serverConfig.ControlConfig.FlannelExternalIP = cfg.FlannelExternalIP
 	serverConfig.ControlConfig.EgressSelectorMode = cfg.EgressSelectorMode

pkg/daemons/config/types.go

@@ -49,6 +49,7 @@ type Node struct {
 	FlannelConfFile          string
 	FlannelConfOverride      bool
 	FlannelIface             *net.Interface
+	FlannelIPv4NoMasq        bool
 	FlannelIPv6Masq          bool
 	FlannelExternalIP        bool
 	EgressSelectorMode       string
@@ -180,6 +181,7 @@ type CriticalControlArgs struct {
 	EncryptSecrets        bool         `cli:"secrets-encryption"`
 	EmbeddedRegistry      bool         `cli:"embedded-registry"`
 	FlannelBackend        string       `cli:"flannel-backend"`
+	FlannelIPv4NoMasq     bool         `cli:"flannel-ipv4-no-masq"`
 	FlannelIPv6Masq       bool         `cli:"flannel-ipv6-masq"`
 	FlannelExternalIP     bool         `cli:"flannel-external-ip"`
 	EgressSelectorMode    string       `cli:"egress-selector-mode"`