Skip to content

Commit

Permalink
chores: SWIFT v2 routing changes (#2342)
Browse files Browse the repository at this point in the history 
* chores: chage nodeCIDRs to infraVNETCIDRs

* chores: add routes for virtualGW

* fix: add /32 prefix

* fix: manually fmt.Sprintf /32 prefix with virtual gw
  • Loading branch information
@nddq
nddq authored Nov 3, 2023
1 parent d1a03ba commit 2dfc05a
Show file tree
Hide file tree
Showing 5 changed files with 76 additions and 51 deletions.
22 changes: 11 additions & 11 deletions cns/configuration/env.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ const (
// LabelNodeSwiftV2 is the Node label for Swift V2
LabelNodeSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled"
// LabelPodSwiftV2 is the Pod label for Swift V2
LabelPodSwiftV2 = "kubernetes.azure.com/pod-network"
EnvPodCIDRs = "POD_CIDRs"
EnvServiceCIDRs = "SERVICE_CIDRs"
EnvNodeCIDRs = "NODE_CIDRs"
LabelPodSwiftV2 = "kubernetes.azure.com/pod-network"
EnvPodCIDRs = "POD_CIDRs"
EnvServiceCIDRs = "SERVICE_CIDRs"
EnvInfraVNETCIDRs = "INFRA_VNET_CIDRs"
)

// ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment.
Expand All @@ -29,8 +29,8 @@ var ErrPodCIDRsUnset = errors.Errorf("must declare %s environment variable", Env
// ErrServiceCIDRsUnset indicates the the $EnvServiceCIDRs variable is unset in the environment.
var ErrServiceCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDRs)

// ErrNodeCIDRsUnset indicates the the $EnvNodeCIDRs variable is unset in the environment.
var ErrNodeCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvNodeCIDRs)
// ErrInfraVNETCIDRsUnset indicates the the $EnvInfraVNETCIDRs variable is unset in the environment.
var ErrInfraVNETCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvInfraVNETCIDRs)

// NodeName checks the environment variables for the NODENAME and returns it or an error if unset.
func NodeName() (string, error) {
Expand Down Expand Up @@ -62,10 +62,10 @@ func ServiceCIDRs() (string, error) {
return serviceCIDRs, nil
}

func NodeCIDRs() (string, error) {
nodeCIDRs := os.Getenv(EnvNodeCIDRs)
if nodeCIDRs == "" {
return "", ErrNodeCIDRsUnset
func InfraVNETCIDRs() (string, error) {
infraVNETCIDRs := os.Getenv(EnvInfraVNETCIDRs)
if infraVNETCIDRs == "" {
return "", ErrInfraVNETCIDRsUnset
}
return nodeCIDRs, nil
return infraVNETCIDRs, nil
}
10 changes: 10 additions & 0 deletions cns/configuration/env_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,13 @@ func TestServiceCIDRs(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, "test", cidr)
}

func TestInfraVNETCIDRs(t *testing.T) {
_, err := InfraVNETCIDRs()
require.Error(t, err)
require.ErrorIs(t, err, ErrInfraVNETCIDRsUnset)
os.Setenv(EnvInfraVNETCIDRs, "test")
cidr, err := InfraVNETCIDRs()
assert.NoError(t, err)
assert.Equal(t, "test", cidr)
}
45 changes: 25 additions & 20 deletions cns/middlewares/mock/mockSWIFTv2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ var (
const (
prefixLength = 32
overlayGatewayv4 = "169.254.1.1"
virtualGW = "169.254.2.1"
overlayGatewayV6 = "fe80::1234:5678:9abc"
)

Expand Down Expand Up @@ -56,7 +57,7 @@ func (m *SWIFTv2Middleware) SetMTPNCReady() {
func (m *SWIFTv2Middleware) SetEnvVar() {
os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24")
os.Setenv(configuration.EnvServiceCIDRs, "10.0.2.10/24")
os.Setenv(configuration.EnvNodeCIDRs, "10.0.3.10/24")
os.Setenv(configuration.EnvInfraVNETCIDRs, "10.0.3.10/24")
}

func (m *SWIFTv2Middleware) UnsetEnvVar() error {
Expand All @@ -66,8 +67,8 @@ func (m *SWIFTv2Middleware) UnsetEnvVar() error {
if err := os.Unsetenv(configuration.EnvServiceCIDRs); err != nil {
return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvServiceCIDRs, err)
}
if err := os.Unsetenv(configuration.EnvNodeCIDRs); err != nil {
return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvNodeCIDRs, err)
if err := os.Unsetenv(configuration.EnvInfraVNETCIDRs); err != nil {
return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvInfraVNETCIDRs, err)
}
return nil
}
Expand Down Expand Up @@ -130,20 +131,24 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error {
podIPInfo.Routes = []cns.Route{}
switch podIPInfo.NICType {
case cns.DelegatedVMNIC:
virtualGWRoute := cns.Route{
IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength),
}
// default route via SWIFT v2 interface
route := cns.Route{
IPAddress: "0.0.0.0/0",
IPAddress: "0.0.0.0/0",
GatewayIPAddress: virtualGW,
}
podIPInfo.Routes = []cns.Route{route}
podIPInfo.Routes = []cns.Route{virtualGWRoute, route}
case cns.InfraNIC:
// Get and parse nodeCIDRs from env
nodeCIDRs, err := configuration.NodeCIDRs()
// Get and parse infraVNETCIDRs from env
infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get nodeCIDR from env")
return errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
}
nodeCIDRsv4, nodeCIDRsv6, err := utils.ParseCIDRs(nodeCIDRs)
infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse nodeCIDRs")
return errors.Wrapf(err, "failed to parse infraVNETCIDRs")
}

// Get and parse podCIDRs from env
Expand Down Expand Up @@ -187,13 +192,13 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error {
}
podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route)
}
// route for IPv4 nodeCIDR traffic
for _, nodeCIDRv4 := range nodeCIDRsv4 {
nodeCIDRv4Route := cns.Route{
IPAddress: nodeCIDRv4,
// route for IPv4 infraVNETCIDR traffic
for _, infraVNETCIDRsv4 := range infraVNETCIDRsv4 {
infraVNETCIDRsv4Route := cns.Route{
IPAddress: infraVNETCIDRsv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv4Route)
podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRsv4Route)
}
} else {
// routes for IPv6 podCIDR traffic
Expand All @@ -212,13 +217,13 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error {
}
podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route)
}
// route for IPv6 nodeCIDR traffic
for _, nodeCIDRv6 := range nodeCIDRsv6 {
nodeCIDRv6Route := cns.Route{
IPAddress: nodeCIDRv6,
// route for IPv6 infraVNETCIDR traffic
for _, infraVNETCIDRv6 := range infraVNETCIDRsv6 {
infraVNETCIDRv6Route := cns.Route{
IPAddress: infraVNETCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv6Route)
podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv6Route)
}
}
podIPInfo.SkipDefaultRoutes = true
Expand Down
39 changes: 22 additions & 17 deletions cns/middlewares/swiftV2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ var (
const (
prefixLength = 32
overlayGatewayv4 = "169.254.1.1"
virtualGW = "169.254.2.1"
overlayGatewayV6 = "fe80::1234:5678:9abc"
)

Expand Down Expand Up @@ -114,20 +115,24 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error {
podIPInfo.Routes = []cns.Route{}
switch podIPInfo.NICType {
case cns.DelegatedVMNIC:
virtualGWRoute := cns.Route{
IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength),
}
// default route via SWIFT v2 interface
route := cns.Route{
IPAddress: "0.0.0.0/0",
IPAddress: "0.0.0.0/0",
GatewayIPAddress: virtualGW,
}
podIPInfo.Routes = []cns.Route{route}
podIPInfo.Routes = []cns.Route{virtualGWRoute, route}
case cns.InfraNIC:
// Get and parse nodeCIDRs from env
nodeCIDRs, err := configuration.NodeCIDRs()
// Get and parse infraVNETCIDRs from env
infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get nodeCIDR from env")
return errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
}
nodeCIDRsv4, nodeCIDRsv6, err := utils.ParseCIDRs(nodeCIDRs)
infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse nodeCIDRs")
return errors.Wrapf(err, "failed to parse infraVNETCIDRs")
}

// Get and parse podCIDRs from env
Expand Down Expand Up @@ -171,13 +176,13 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error {
}
podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route)
}
// route for IPv4 nodeCIDR traffic
for _, nodeCIDRv4 := range nodeCIDRsv4 {
nodeCIDRv4Route := cns.Route{
IPAddress: nodeCIDRv4,
// route for IPv4 infraVNETCIDR traffic
for _, infraVNETCIDRv4 := range infraVNETCIDRsv4 {
infraVNETCIDRv4Route := cns.Route{
IPAddress: infraVNETCIDRv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv4Route)
podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv4Route)
}
} else {
// routes for IPv6 podCIDR traffic
Expand All @@ -196,13 +201,13 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error {
}
podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route)
}
// route for IPv6 nodeCIDR traffic
for _, nodeCIDRv6 := range nodeCIDRsv6 {
nodeCIDRv6Route := cns.Route{
IPAddress: nodeCIDRv6,
// route for IPv6 infraVNETCIDR traffic
for _, infraVNETCIDRv6 := range infraVNETCIDRsv6 {
infraVNETCIDRv6Route := cns.Route{
IPAddress: infraVNETCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv6Route)
podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv6Route)
}
}
podIPInfo.SkipDefaultRoutes = true
Expand Down
11 changes: 8 additions & 3 deletions cns/middlewares/swiftV2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package middlewares

import (
"context"
"fmt"
"os"
"testing"

Expand Down Expand Up @@ -30,13 +31,13 @@ var (
func setEnvVar() {
os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32")
os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16,16A0:0010:AB00:0000::/32")
os.Setenv(configuration.EnvNodeCIDRs, "10.240.0.1/16,16A0:0020:AB00:0000::/32")
os.Setenv(configuration.EnvInfraVNETCIDRs, "10.240.0.1/16,16A0:0020:AB00:0000::/32")
}

func unsetEnvVar() {
os.Unsetenv(configuration.EnvPodCIDRs)
os.Unsetenv(configuration.EnvServiceCIDRs)
os.Unsetenv(configuration.EnvNodeCIDRs)
os.Unsetenv(configuration.EnvInfraVNETCIDRs)
}

func TestMain(m *testing.M) {
Expand Down Expand Up @@ -200,7 +201,11 @@ func TestSetRoutesSuccess(t *testing.T) {
MacAddress: "12:34:56:78:9a:bc",
Routes: []cns.Route{
{
IPAddress: "0.0.0.0/0",
IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength),
},
{
IPAddress: "0.0.0.0/0",
GatewayIPAddress: virtualGW,
},
},
},
Expand Down

0 comments on commit 2dfc05a

Please sign in to comment.