From 809fdd9fe1cf0f3102a84dc4ad679ad3c9169f70 Mon Sep 17 00:00:00 2001 From: Antoni Segura Puimedon Date: Fri, 13 Mar 2020 09:05:27 +0100 Subject: [PATCH] MCD: Add node-ip subcommand node-ip is a subcommand that allows the user to see which IP should the node use in cases of multiple interface and multiple address nodes. This is useful to prevent cases where Container Runtime related services bind to an interface that is not reachable in the control plane. It has two commands: * show: Takes one or more Virtual IPs of the control plane and it gives you one eligible IP on stdout. * set: Takes one or more Virtual IPs of the control plane and sets systemd service configuration for services like CRI-O or Kubelet that need to bind to the control plane. Signed-off-by: Antoni Segura Puimedon --- cmd/machine-config-daemon/node-ip.go | 150 +++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 2 - pkg/daemon/nodenet/utils.go | 141 +++++++++++++++++++++++++ 4 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 cmd/machine-config-daemon/node-ip.go create mode 100644 pkg/daemon/nodenet/utils.go diff --git a/cmd/machine-config-daemon/node-ip.go b/cmd/machine-config-daemon/node-ip.go new file mode 100644 index 0000000000..ca2cf47b75 --- /dev/null +++ b/cmd/machine-config-daemon/node-ip.go @@ -0,0 +1,150 @@ +package main + +import ( + "flag" + "fmt" + "net" + "os" + "path/filepath" + + // Enable sha256 in container image references + _ "crypto/sha256" + + "github.com/golang/glog" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/openshift/machine-config-operator/pkg/daemon/nodenet" +) + +const ( + kubeletSvcOverridePath = "/etc/systemd/system/kubelet.service.d/20-nodenet.conf" + crioSvcOverridePath = "/etc/systemd/system/crio.service.d/20-nodenet.conf" +) + +var nodeIPCmd = &cobra.Command{ + Use: "node-ip", + DisableFlagsInUseLine: true, + Short: "Node IP tools", + Long: "Node IP has tools that aid in the configuration of nodes in platforms that use Virtual IPs", +} + +var nodeIPShowCmd = &cobra.Command{ + Use: "show", + DisableFlagsInUseLine: true, + Short: "Show a configured IP address that directly routes to the given Virtual IPs", + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + err := show(cmd, args) + if err != nil { + glog.Exitf("error in node-ip show: %v\n", err) + } + }, +} + +var nodeIPSetCmd = &cobra.Command{ + Use: "set", + DisableFlagsInUseLine: true, + Short: "Sets container runtime services to bind to a configured IP address that directly routes to the given virtual IPs", + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + err := set(cmd, args) + if err != nil { + glog.Exitf("error in node-ip set: %v\n", err) + } + }, +} + +// init executes upon import +func init() { + rootCmd.AddCommand(nodeIPCmd) + nodeIPCmd.AddCommand(nodeIPShowCmd) + nodeIPCmd.AddCommand(nodeIPSetCmd) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + flag.Set("logtostderr", "true") + flag.Parse() +} + +func show(_ *cobra.Command, args []string) error { + vips := make([]net.IP, len(args)) + for i, arg := range args { + vips[i] = net.ParseIP(arg) + if vips[i] == nil { + return fmt.Errorf("Failed to parse IP address %s", arg) + } + glog.V(3).Infof("Parsed Virtual IP %s", vips[i]) + } + + nodeAddrs, err := nodenet.AddressesRouting(vips, nodenet.NonDeprecatedAddress, nodenet.NonDefaultRoute) + if err != nil { + return err + } + + if len(nodeAddrs) > 0 { + fmt.Println(nodeAddrs[0]) + } else { + return fmt.Errorf("Failed to find node IP") + } + + return nil +} + +func set(_ *cobra.Command, args []string) error { + vips := make([]net.IP, len(args)) + for i, arg := range args { + vips[i] = net.ParseIP(arg) + if vips[i] == nil { + return fmt.Errorf("Failed to parse IP address %s", arg) + } + glog.V(3).Infof("Parsed Virtual IP %s", vips[i]) + } + + nodeAddrs, err := nodenet.AddressesRouting(vips, nodenet.NonDeprecatedAddress, nodenet.NonDefaultRoute) + if err != nil { + return err + } + + var chosenAddress net.IP + if len(nodeAddrs) > 0 { + chosenAddress = nodeAddrs[0] + } else { + return fmt.Errorf("Failed to find node IP") + } + glog.Infof("Chosen Node IP %s", chosenAddress) + + // Kubelet + glog.V(2).Infof("Opening Kubelet service override path %s", kubeletSvcOverridePath) + kOverride, err := os.Create(kubeletSvcOverridePath) + if err != nil { + return err + } + defer kOverride.Close() + + kOverrideContent := fmt.Sprintf("[Service]\nEnvironment=\"KUBELET_NODE_IP=%s\"\n", chosenAddress) + glog.V(3).Infof("Writing Kubelet service override with content %s", kOverrideContent) + _, err = kOverride.WriteString(kOverrideContent) + if err != nil { + return err + } + + // CRI-O + crioOverrideDir := filepath.Dir(crioSvcOverridePath) + err = os.MkdirAll(crioOverrideDir, 0755) + if err != nil { + return err + } + glog.V(2).Infof("Opening CRI-O service override path %s", crioSvcOverridePath) + cOverride, err := os.Create(crioSvcOverridePath) + if err != nil { + return err + } + defer cOverride.Close() + + cOverrideContent := fmt.Sprintf("[Service]\nEnvironment=\"CONTAINER_STREAM_ADDRESS=%s\"\n", chosenAddress) + glog.V(3).Infof("Writing CRI-O service override with content %s", cOverrideContent) + _, err = cOverride.WriteString(cOverrideContent) + if err != nil { + return err + } + return nil +} diff --git a/go.mod b/go.mod index b06779c271..17108fc6c2 100644 --- a/go.mod +++ b/go.mod @@ -50,9 +50,10 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb + github.com/vishvananda/netlink v1.0.0 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 // indirect - golang.org/x/sys v0.0.0-20191002091554-b397fe3ad8ed // indirect + golang.org/x/sys v0.0.0-20191002091554-b397fe3ad8ed golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 gonum.org/v1/gonum v0.0.0-20190929233944-b20cf7805fc4 // indirect gonum.org/v1/netlib v0.0.0-20190926062253-2d6e29b73a19 // indirect diff --git a/go.sum b/go.sum index 3e8975e6e3..92af49706e 100644 --- a/go.sum +++ b/go.sum @@ -713,8 +713,6 @@ github.com/opencontainers/selinux v1.3.0 h1:xsI95WzPZu5exzA6JzkLSfdr/DilzOhCJOqG github.com/opencontainers/selinux v1.3.0/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52 h1:B8hYj3NxHmjsC3T+tnlZ1UhInqUgnyF1zlGPmzNg2Qk= github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= -github.com/openshift/api v0.0.0-20200116145750-0e2ff1e215dd h1:WIrzR6PXOptxWGafidO/zMixrHDITEBHdz9k9AkAL1U= -github.com/openshift/api v0.0.0-20200116145750-0e2ff1e215dd/go.mod h1:fT6U/JfG8uZzemTRwZA2kBDJP5nWz7v05UHnty/D+pk= github.com/openshift/api v0.0.0-20200302180901-b4f75e525601 h1:w+BZAqw/cIQe03ilt0Za/4wnERpbtTukVjCITh1vsO4= github.com/openshift/api v0.0.0-20200302180901-b4f75e525601/go.mod h1:frTMT4l3rOMlXj3ClYgKxgkq24D7IKXb3Bl4vJEewJw= github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc= diff --git a/pkg/daemon/nodenet/utils.go b/pkg/daemon/nodenet/utils.go new file mode 100644 index 0000000000..ee2b922538 --- /dev/null +++ b/pkg/daemon/nodenet/utils.go @@ -0,0 +1,141 @@ +package nodenet + +import ( + "net" + + "github.com/golang/glog" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +// AddressFilter is a function type to filter addresses +type AddressFilter func(netlink.Addr) bool + +// RouteFilter is a function type to filter routes +type RouteFilter func(netlink.Route) bool + +func getAddrs() (addrMap map[netlink.Link][]netlink.Addr, err error) { + nlHandle, err := netlink.NewHandle() + if err != nil { + return nil, err + } + defer nlHandle.Delete() + + links, err := nlHandle.LinkList() + if err != nil { + return nil, err + } + + addrMap = make(map[netlink.Link][]netlink.Addr) + for _, link := range links { + addresses, err := nlHandle.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return nil, err + } + for _, address := range addresses { + if _, ok := addrMap[link]; ok { + addrMap[link] = append(addrMap[link], address) + } else { + addrMap[link] = []netlink.Addr{address} + } + } + } + glog.V(7).Infof("retrieved Address map %+v", addrMap) + return addrMap, nil +} + +func getRouteMap() (routeMap map[int][]netlink.Route, err error) { + nlHandle, err := netlink.NewHandle() + if err != nil { + return nil, err + } + defer nlHandle.Delete() + + routes, err := nlHandle.RouteList(nil, netlink.FAMILY_V6) + if err != nil { + return nil, err + } + + routeMap = make(map[int][]netlink.Route) + for _, route := range routes { + if route.Protocol != unix.RTPROT_RA { + glog.V(4).Infof("Ignoring route non Router advertisement route %+v", route) + continue + } + if _, ok := routeMap[route.LinkIndex]; ok { + routeMap[route.LinkIndex] = append(routeMap[route.LinkIndex], route) + } else { + routeMap[route.LinkIndex] = []netlink.Route{route} + } + } + + glog.V(7).Infof("Retrieved IPv6 route map %+v", routeMap) + + return routeMap, nil +} + +// NonDeprecatedAddress returns true if the address is IPv6 and has a preferred lifetime of 0 +func NonDeprecatedAddress(address netlink.Addr) bool { + return !(net.IPv6len == len(address.IP) && address.PreferedLft == 0) +} + +// NonDefaultRoute returns whether the passed Route is the default +func NonDefaultRoute(route netlink.Route) bool { + return route.Dst != nil +} + +// AddressesRouting takes a slice of Virtual IPs and returns a slice of configured addresses in the current network namespace that directly route to those vips. You can optionally pass an AddressFilter and/or RouteFilter to further filter down which addresses are considered +func AddressesRouting(vips []net.IP, af AddressFilter, rf RouteFilter) ([]net.IP, error) { + addrMap, err := getAddrs() + if err != nil { + return nil, err + } + + var routeMap map[int][]netlink.Route + matches := make([]net.IP, 0) + for link, addresses := range addrMap { + for _, address := range addresses { + maskPrefix, maskBits := address.Mask.Size() + if !af(address) { + continue + } + if net.IPv6len == len(address.IP) && maskPrefix == maskBits { + if routeMap == nil { + routeMap, err = getRouteMap() + if err != nil { + panic(err) + } + } + if routes, ok := routeMap[link.Attrs().Index]; ok { + for _, route := range routes { + if !rf(route) { + continue + } + routePrefix, _ := route.Dst.Mask.Size() + glog.V(4).Infof("Checking route %+v (mask %s) for address %+v", route, route.Dst.Mask, address) + if routePrefix != 0 { + containmentNet := net.IPNet{IP: address.IP, Mask: route.Dst.Mask} + for _, vip := range vips { + glog.V(3).Infof("Checking whether address %s with route %s contains VIP %s", address, route, vip) + if containmentNet.Contains(vip) { + glog.V(2).Infof("Address %s with route %s contains VIP %s", address, route, vip) + matches = append(matches, address.IP) + } + } + } + } + } + } else { + for _, vip := range vips { + glog.V(3).Infof("Checking whether address %s contains VIP %s", address, vip) + if address.Contains(vip) { + glog.V(2).Infof("Address %s contains VIP %s", address, vip) + matches = append(matches, address.IP) + } + } + } + } + + } + return matches, nil +}