diff --git a/cmd/machine-config-daemon/node-ip.go b/cmd/machine-config-daemon/node-ip.go new file mode 100644 index 0000000000..ca2cf47b75 --- /dev/null +++ b/cmd/machine-config-daemon/node-ip.go @@ -0,0 +1,150 @@ +package main + +import ( + "flag" + "fmt" + "net" + "os" + "path/filepath" + + // Enable sha256 in container image references + _ "crypto/sha256" + + "github.com/golang/glog" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/openshift/machine-config-operator/pkg/daemon/nodenet" +) + +const ( + kubeletSvcOverridePath = "/etc/systemd/system/kubelet.service.d/20-nodenet.conf" + crioSvcOverridePath = "/etc/systemd/system/crio.service.d/20-nodenet.conf" +) + +var nodeIPCmd = &cobra.Command{ + Use: "node-ip", + DisableFlagsInUseLine: true, + Short: "Node IP tools", + Long: "Node IP has tools that aid in the configuration of nodes in platforms that use Virtual IPs", +} + +var nodeIPShowCmd = &cobra.Command{ + Use: "show", + DisableFlagsInUseLine: true, + Short: "Show a configured IP address that directly routes to the given Virtual IPs", + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + err := show(cmd, args) + if err != nil { + glog.Exitf("error in node-ip show: %v\n", err) + } + }, +} + +var nodeIPSetCmd = &cobra.Command{ + Use: "set", + DisableFlagsInUseLine: true, + Short: "Sets container runtime services to bind to a configured IP address that directly routes to the given virtual IPs", + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + err := set(cmd, args) + if err != nil { + glog.Exitf("error in node-ip set: %v\n", err) + } + }, +} + +// init executes upon import +func init() { + rootCmd.AddCommand(nodeIPCmd) + nodeIPCmd.AddCommand(nodeIPShowCmd) + nodeIPCmd.AddCommand(nodeIPSetCmd) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + flag.Set("logtostderr", "true") + flag.Parse() +} + +func show(_ *cobra.Command, args []string) error { + vips := make([]net.IP, len(args)) + for i, arg := range args { + vips[i] = net.ParseIP(arg) + if vips[i] == nil { + return fmt.Errorf("Failed to parse IP address %s", arg) + } + glog.V(3).Infof("Parsed Virtual IP %s", vips[i]) + } + + nodeAddrs, err := nodenet.AddressesRouting(vips, nodenet.NonDeprecatedAddress, nodenet.NonDefaultRoute) + if err != nil { + return err + } + + if len(nodeAddrs) > 0 { + fmt.Println(nodeAddrs[0]) + } else { + return fmt.Errorf("Failed to find node IP") + } + + return nil +} + +func set(_ *cobra.Command, args []string) error { + vips := make([]net.IP, len(args)) + for i, arg := range args { + vips[i] = net.ParseIP(arg) + if vips[i] == nil { + return fmt.Errorf("Failed to parse IP address %s", arg) + } + glog.V(3).Infof("Parsed Virtual IP %s", vips[i]) + } + + nodeAddrs, err := nodenet.AddressesRouting(vips, nodenet.NonDeprecatedAddress, nodenet.NonDefaultRoute) + if err != nil { + return err + } + + var chosenAddress net.IP + if len(nodeAddrs) > 0 { + chosenAddress = nodeAddrs[0] + } else { + return fmt.Errorf("Failed to find node IP") + } + glog.Infof("Chosen Node IP %s", chosenAddress) + + // Kubelet + glog.V(2).Infof("Opening Kubelet service override path %s", kubeletSvcOverridePath) + kOverride, err := os.Create(kubeletSvcOverridePath) + if err != nil { + return err + } + defer kOverride.Close() + + kOverrideContent := fmt.Sprintf("[Service]\nEnvironment=\"KUBELET_NODE_IP=%s\"\n", chosenAddress) + glog.V(3).Infof("Writing Kubelet service override with content %s", kOverrideContent) + _, err = kOverride.WriteString(kOverrideContent) + if err != nil { + return err + } + + // CRI-O + crioOverrideDir := filepath.Dir(crioSvcOverridePath) + err = os.MkdirAll(crioOverrideDir, 0755) + if err != nil { + return err + } + glog.V(2).Infof("Opening CRI-O service override path %s", crioSvcOverridePath) + cOverride, err := os.Create(crioSvcOverridePath) + if err != nil { + return err + } + defer cOverride.Close() + + cOverrideContent := fmt.Sprintf("[Service]\nEnvironment=\"CONTAINER_STREAM_ADDRESS=%s\"\n", chosenAddress) + glog.V(3).Infof("Writing CRI-O service override with content %s", cOverrideContent) + _, err = cOverride.WriteString(cOverrideContent) + if err != nil { + return err + } + return nil +} diff --git a/go.mod b/go.mod index b06779c271..17108fc6c2 100644 --- a/go.mod +++ b/go.mod @@ -50,9 +50,10 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb + github.com/vishvananda/netlink v1.0.0 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 // indirect - golang.org/x/sys v0.0.0-20191002091554-b397fe3ad8ed // indirect + golang.org/x/sys v0.0.0-20191002091554-b397fe3ad8ed golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 gonum.org/v1/gonum v0.0.0-20190929233944-b20cf7805fc4 // indirect gonum.org/v1/netlib v0.0.0-20190926062253-2d6e29b73a19 // indirect diff --git a/go.sum b/go.sum index 3e8975e6e3..92af49706e 100644 --- a/go.sum +++ b/go.sum @@ -713,8 +713,6 @@ github.com/opencontainers/selinux v1.3.0 h1:xsI95WzPZu5exzA6JzkLSfdr/DilzOhCJOqG github.com/opencontainers/selinux v1.3.0/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52 h1:B8hYj3NxHmjsC3T+tnlZ1UhInqUgnyF1zlGPmzNg2Qk= github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= -github.com/openshift/api v0.0.0-20200116145750-0e2ff1e215dd h1:WIrzR6PXOptxWGafidO/zMixrHDITEBHdz9k9AkAL1U= -github.com/openshift/api v0.0.0-20200116145750-0e2ff1e215dd/go.mod h1:fT6U/JfG8uZzemTRwZA2kBDJP5nWz7v05UHnty/D+pk= github.com/openshift/api v0.0.0-20200302180901-b4f75e525601 h1:w+BZAqw/cIQe03ilt0Za/4wnERpbtTukVjCITh1vsO4= github.com/openshift/api v0.0.0-20200302180901-b4f75e525601/go.mod h1:frTMT4l3rOMlXj3ClYgKxgkq24D7IKXb3Bl4vJEewJw= github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc= diff --git a/pkg/daemon/nodenet/utils.go b/pkg/daemon/nodenet/utils.go new file mode 100644 index 0000000000..ee2b922538 --- /dev/null +++ b/pkg/daemon/nodenet/utils.go @@ -0,0 +1,141 @@ +package nodenet + +import ( + "net" + + "github.com/golang/glog" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +// AddressFilter is a function type to filter addresses +type AddressFilter func(netlink.Addr) bool + +// RouteFilter is a function type to filter routes +type RouteFilter func(netlink.Route) bool + +func getAddrs() (addrMap map[netlink.Link][]netlink.Addr, err error) { + nlHandle, err := netlink.NewHandle() + if err != nil { + return nil, err + } + defer nlHandle.Delete() + + links, err := nlHandle.LinkList() + if err != nil { + return nil, err + } + + addrMap = make(map[netlink.Link][]netlink.Addr) + for _, link := range links { + addresses, err := nlHandle.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return nil, err + } + for _, address := range addresses { + if _, ok := addrMap[link]; ok { + addrMap[link] = append(addrMap[link], address) + } else { + addrMap[link] = []netlink.Addr{address} + } + } + } + glog.V(7).Infof("retrieved Address map %+v", addrMap) + return addrMap, nil +} + +func getRouteMap() (routeMap map[int][]netlink.Route, err error) { + nlHandle, err := netlink.NewHandle() + if err != nil { + return nil, err + } + defer nlHandle.Delete() + + routes, err := nlHandle.RouteList(nil, netlink.FAMILY_V6) + if err != nil { + return nil, err + } + + routeMap = make(map[int][]netlink.Route) + for _, route := range routes { + if route.Protocol != unix.RTPROT_RA { + glog.V(4).Infof("Ignoring route non Router advertisement route %+v", route) + continue + } + if _, ok := routeMap[route.LinkIndex]; ok { + routeMap[route.LinkIndex] = append(routeMap[route.LinkIndex], route) + } else { + routeMap[route.LinkIndex] = []netlink.Route{route} + } + } + + glog.V(7).Infof("Retrieved IPv6 route map %+v", routeMap) + + return routeMap, nil +} + +// NonDeprecatedAddress returns true if the address is IPv6 and has a preferred lifetime of 0 +func NonDeprecatedAddress(address netlink.Addr) bool { + return !(net.IPv6len == len(address.IP) && address.PreferedLft == 0) +} + +// NonDefaultRoute returns whether the passed Route is the default +func NonDefaultRoute(route netlink.Route) bool { + return route.Dst != nil +} + +// AddressesRouting takes a slice of Virtual IPs and returns a slice of configured addresses in the current network namespace that directly route to those vips. You can optionally pass an AddressFilter and/or RouteFilter to further filter down which addresses are considered +func AddressesRouting(vips []net.IP, af AddressFilter, rf RouteFilter) ([]net.IP, error) { + addrMap, err := getAddrs() + if err != nil { + return nil, err + } + + var routeMap map[int][]netlink.Route + matches := make([]net.IP, 0) + for link, addresses := range addrMap { + for _, address := range addresses { + maskPrefix, maskBits := address.Mask.Size() + if !af(address) { + continue + } + if net.IPv6len == len(address.IP) && maskPrefix == maskBits { + if routeMap == nil { + routeMap, err = getRouteMap() + if err != nil { + panic(err) + } + } + if routes, ok := routeMap[link.Attrs().Index]; ok { + for _, route := range routes { + if !rf(route) { + continue + } + routePrefix, _ := route.Dst.Mask.Size() + glog.V(4).Infof("Checking route %+v (mask %s) for address %+v", route, route.Dst.Mask, address) + if routePrefix != 0 { + containmentNet := net.IPNet{IP: address.IP, Mask: route.Dst.Mask} + for _, vip := range vips { + glog.V(3).Infof("Checking whether address %s with route %s contains VIP %s", address, route, vip) + if containmentNet.Contains(vip) { + glog.V(2).Infof("Address %s with route %s contains VIP %s", address, route, vip) + matches = append(matches, address.IP) + } + } + } + } + } + } else { + for _, vip := range vips { + glog.V(3).Infof("Checking whether address %s contains VIP %s", address, vip) + if address.Contains(vip) { + glog.V(2).Infof("Address %s contains VIP %s", address, vip) + matches = append(matches, address.IP) + } + } + } + } + + } + return matches, nil +}