feat: convert execs to ip to netlink calls

pkg/controllers/proxy/network_services_controller.go

@@ -33,6 +33,7 @@ const (
 	KubeDummyIf       = "kube-dummy-if"
 	KubeTunnelIfv4    = "kube-tunnel-if"
 	KubeTunnelIfv6    = "kube-tunnel-v6"
+	KubeBridgeIf      = "kube-bridge"
 	IfaceNotFound     = "Link not found"
 	IfaceHasAddr      = "file exists"
 	IfaceHasNoAddr    = "cannot assign requested address"
@@ -42,11 +43,13 @@ const (
 	IpvsSvcFSched2    = "flag-2"
 	IpvsSvcFSched3    = "flag-3"
 
-	customDSRRouteTableID    = "78"
-	customDSRRouteTableName  = "kube-router-dsr"
-	externalIPRouteTableID   = "79"
-	externalIPRouteTableName = "external_ip"
-	kubeRouterProxyName      = "kube-router"
+	customDSRRouteTableID              = 78
+	customDSRRouteTableName            = "kube-router-dsr"
+	externalIPRouteTableID             = 79
+	externalIPRouteTableName           = "external_ip"
+	kubeRouterProxyName                = "kube-router"
+	defaultTrafficDirectorRulePriority = 32764
+	defaultDSRPolicyRulePriority       = 32765
 
 	// Taken from https://github.com/torvalds/linux/blob/master/include/uapi/linux/ip_vs.h#L21
 	ipvsPersistentFlagHex = 0x0001
@@ -1727,24 +1730,38 @@ func (nsc *NetworkServicesController) cleanupMangleTableRule(ip string, protocol
 // For DSR it is required that we dont assign the VIP to any interface to avoid martian packets
 // http://www.austintek.com/LVS/LVS-HOWTO/HOWTO/LVS-HOWTO.routing_to_VIP-less_director.html
 // routeVIPTrafficToDirector: setups policy routing so that FWMARKed packets are delivered locally
-func routeVIPTrafficToDirector(fwmark string, family v1.IPFamily) error {
-	ipArgs := make([]string, 0)
+func routeVIPTrafficToDirector(fwmark uint32, family v1.IPFamily) error {
+	nFamily := netlink.FAMILY_V4
 	if family == v1.IPv6Protocol {
-		ipArgs = append(ipArgs, "-6")
+		nFamily = netlink.FAMILY_V6
 	}
 
-	out, err := runIPCommandsWithArgs(ipArgs, "rule", "list").Output()
+	nRule := netlink.NewRule()
+	nRule.Mark = fwmark
+	nRule.Table = customDSRRouteTableID
+	nRule.Priority = defaultTrafficDirectorRulePriority
+
+	routes, err := netlink.RuleListFiltered(nFamily, nRule,
+		netlink.RT_FILTER_MARK|netlink.RT_FILTER_TABLE|netlink.RT_FILTER_PRIORITY)
 	if err != nil {
-		return errors.New("Failed to verify if `ip rule` exists due to: " + err.Error())
+		return fmt.Errorf("failed to verify if `ip rule` exists due to: %v", err)
 	}
-	if !strings.Contains(string(out), fwmark+" ") {
-		err = runIPCommandsWithArgs(ipArgs, "rule", "add", "prio", "32764", "fwmark", fwmark, "table",
-			customDSRRouteTableID).Run()
+
+	if len(routes) < 1 {
+		klog.V(1).Infof("adding policy rule (%s) to lookup traffic to VIP through the custom routing table", nRule)
+		err = netlink.RuleAdd(nRule)
 		if err != nil {
-			return errors.New("Failed to add policy rule to lookup traffic to VIP through the custom " +
-				" routing table due to " + err.Error())
+			return fmt.Errorf("failed to add policy rule to lookup traffic to VIP through the custom "+
+				"routing table due to %v", err)
+		}
+	} else {
+		klog.V(1).Infof("policy rule (%s) for mark %d already exists, skipping", nRule, fwmark)
+		klog.V(1).Info("Routes Found:")
+		for _, route := range routes {
+			klog.V(1).Infof("Route: %+v with mark %d", route, route.Mark)
 		}
 	}
+
 	return nil
 }
 

pkg/controllers/proxy/service_endpoints_sync.go

@@ -557,7 +557,7 @@ func (nsc *NetworkServicesController) setupExternalIPForDSRService(svcIn *servic
 	}
 
 	// do policy routing to deliver the packet locally so that IPVS can pick the packet
-	err = routeVIPTrafficToDirector("0x"+fmt.Sprintf("%x", fwMark), family)
+	err = routeVIPTrafficToDirector(fwMark, family)
 	if err != nil {
 		return fmt.Errorf("failed to setup ip rule to lookup traffic to external IP: %s through custom "+
 			"route table due to %v", externalIP, err)

pkg/controllers/proxy/utils.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"hash/fnv"
 	"net"
-	"os/exec"
 	"runtime"
 	"strconv"
 	"strings"
@@ -603,14 +602,6 @@ func getIPVSFirewallInputChainRule(family v1.IPFamily) []string {
 		"-j", ipvsFirewallChainName}
 }
 
-// runIPCommandsWithArgs extend the exec.Command interface to allow passing an additional array of arguments to ip
-func runIPCommandsWithArgs(ipArgs []string, additionalArgs ...string) *exec.Cmd {
-	var allArgs []string
-	allArgs = append(allArgs, ipArgs...)
-	allArgs = append(allArgs, additionalArgs...)
-	return exec.Command("ip", allArgs...)
-}
-
 // getLabelFromMap checks the list of passed labels for the service.kubernetes.io/service-proxy-name
 // label and if it exists, returns it otherwise returns an error
 func getLabelFromMap(label string, labels map[string]string) (string, error) {

pkg/controllers/routing/bgp_policies.go

@@ -278,8 +278,8 @@ func (nrc *NetworkRoutingController) addServiceVIPsDefinedSet() error {
 // create a defined set to represent just the host default route
 func (nrc *NetworkRoutingController) addDefaultRouteDefinedSet() error {
 	for setName, defaultRoute := range map[string]string{
-		defaultRouteSet:   "0.0.0.0/0",
-		defaultRouteSetV6: "::/0",
+		defaultRouteSet:   utils.IPv4DefaultRoute,
+		defaultRouteSetV6: utils.IPv6DefaultRoute,
 	} {
 		currentDefinedSet, err := nrc.getDefinedSetFromGoBGP(setName, gobgpapi.DefinedType_PREFIX)
 		if err != nil {

pkg/routes/pbr.go

@@ -2,15 +2,20 @@ package routes
 
 import (
 	"fmt"
-	"os/exec"
-	"strings"
+	"net"
 
 	"github.com/cloudnativelabs/kube-router/v2/pkg/utils"
+	"github.com/vishvananda/netlink"
+)
+
+const (
+	PBRRuleAdd = iota
+	PBRRuleDel
 )
 
 const (
 	// CustomTableID is the ID of the custom, iproute2 routing table that will be used for policy based routing
-	CustomTableID = "77"
+	CustomTableID = 77
 	// CustomTableName is the name of the custom, iproute2 routing table that will be used for policy based routing
 	CustomTableName = "kube-router"
 )
@@ -35,21 +40,54 @@ func NewPolicyBasedRules(nfa utils.NodeFamilyAware, podIPv4CIDRs, podIPv6CIDRs [
 // ipProtocol is the iproute2 protocol specified as a string ("-4" or "-6"). ipOp is the rule operation specified as a
 // string ("add" or "del). The cidr is the IPv4 / IPv6 source CIDR string that when received will be used to lookup
 // routes in a custom table.
-func ipRuleAbstraction(ipProtocol, ipOp, cidr string) error {
-	out, err := exec.Command("ip", ipProtocol, "rule", "list").Output()
+func ipRuleAbstraction(ipFamily int, ipOp int, cidr string) error {
+	_, nSrc, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return fmt.Errorf("failed to parse CIDR: %s", err.Error())
+	}
+
+	nRule := netlink.NewRule()
+	nRule.Family = ipFamily
+	nRule.Src = nSrc
+	nRule.Table = CustomTableID
+
+	// If the rule that we are abstracting has either the src or dst set to a default route, then we need to handle it
+	// differently. For more information, see: https://github.com/vishvananda/netlink/issues/1080
+	// TODO: If the above issue is resolved, some of the below logic can be removed
+	rules := make([]netlink.Rule, 0)
+	isDefaultRoute, err := utils.IsDefaultRoute(nSrc)
 	if err != nil {
-		return fmt.Errorf("failed to verify if `ip rule` exists: %s", err.Error())
+		return fmt.Errorf("failed to check if CIDR is a default route: %v", err)
 	}
 
-	if strings.Contains(string(out), cidr) && ipOp == "del" {
-		err = exec.Command("ip", ipProtocol, "rule", ipOp, "from", cidr, "lookup", CustomTableID).Run()
+	if isDefaultRoute {
+		var tmpRules []netlink.Rule
+		tmpRules, err = netlink.RuleListFiltered(ipFamily, nRule, netlink.RT_FILTER_TABLE)
 		if err != nil {
-			return fmt.Errorf("failed to add ip rule due to: %s", err.Error())
+			return fmt.Errorf("failed to list rules: %s", err.Error())
 		}
-	} else if !strings.Contains(string(out), cidr) && ipOp == "add" {
-		err = exec.Command("ip", ipProtocol, "rule", ipOp, "from", cidr, "lookup", CustomTableID).Run()
+
+		// Check if one or more of the rules returned are a default route rule
+		for _, rule := range tmpRules {
+			// If the rule has no src, then it is a default route rule which is the match criteria for the rule
+			if rule.Src == nil {
+				rules = append(rules, rule)
+			}
+		}
+	} else {
+		rules, err = netlink.RuleListFiltered(ipFamily, nRule, netlink.RT_FILTER_SRC|netlink.RT_FILTER_TABLE)
 		if err != nil {
-			return fmt.Errorf("failed to add ip rule due to: %s", err.Error())
+			return fmt.Errorf("failed to list rules: %s", err.Error())
+		}
+	}
+
+	if ipOp == PBRRuleDel && len(rules) > 0 {
+		if err := netlink.RuleDel(nRule); err != nil {
+			return fmt.Errorf("failed to delete rule: %s", err.Error())
+		}
+	} else if ipOp == PBRRuleAdd && len(rules) < 1 {
+		if err := netlink.RuleAdd(nRule); err != nil {
+			return fmt.Errorf("failed to add rule: %s", err.Error())
 		}
 	}
 
@@ -66,14 +104,14 @@ func (pbr *PolicyBasedRules) Enable() error {
 
 	if pbr.nfa.IsIPv4Capable() {
 		for _, ipv4CIDR := range pbr.podIPv4CIDRs {
-			if err := ipRuleAbstraction("-4", "add", ipv4CIDR); err != nil {
+			if err := ipRuleAbstraction(netlink.FAMILY_V4, PBRRuleAdd, ipv4CIDR); err != nil {
 				return err
 			}
 		}
 	}
 	if pbr.nfa.IsIPv6Capable() {
 		for _, ipv6CIDR := range pbr.podIPv6CIDRs {
-			if err := ipRuleAbstraction("-6", "add", ipv6CIDR); err != nil {
+			if err := ipRuleAbstraction(netlink.FAMILY_V6, PBRRuleAdd, ipv6CIDR); err != nil {
 				return err
 			}
 		}
@@ -91,14 +129,14 @@ func (pbr *PolicyBasedRules) Disable() error {
 
 	if pbr.nfa.IsIPv4Capable() {
 		for _, ipv4CIDR := range pbr.podIPv4CIDRs {
-			if err := ipRuleAbstraction("-4", "del", ipv4CIDR); err != nil {
+			if err := ipRuleAbstraction(netlink.FAMILY_V4, PBRRuleDel, ipv4CIDR); err != nil {
 				return err
 			}
 		}
 	}
 	if pbr.nfa.IsIPv6Capable() {
 		for _, ipv6CIDR := range pbr.podIPv6CIDRs {
-			if err := ipRuleAbstraction("-6", "del", ipv6CIDR); err != nil {
+			if err := ipRuleAbstraction(netlink.FAMILY_V6, PBRRuleDel, ipv6CIDR); err != nil {
 				return err
 			}
 		}

pkg/tunnels/linux_tunnels.go

@@ -228,14 +228,15 @@ func (o *OverlayTunnel) SetupOverlayTunnel(tunnelName string, nextHop net.IP,
 	// Now that the tunnel link exists, we need to add a route to it, so the node knows where to send traffic bound for
 	// this interface
 	//nolint:gocritic // we understand that we are appending to a new slice
-	cmdArgs := append(ipBase, "route", "list", "table", routes.CustomTableID)
+	cmdArgs := append(ipBase, "route", "list", "table", strconv.Itoa(routes.CustomTableID))
 	out, err = exec.Command("ip", cmdArgs...).CombinedOutput()
 	// This used to be "dev "+tunnelName+" scope" but this isn't consistent with IPv6's output, so we changed it to just
 	// "dev "+tunnelName, but at this point I'm unsure if there was a good reason for adding scope on before, so that's
 	// why this comment is here.
 	if err != nil || !strings.Contains(string(out), "dev "+tunnelName) {
 		//nolint:gocritic // we understand that we are appending to a new slice
-		cmdArgs = append(ipBase, "route", "add", nextHop.String(), "dev", tunnelName, "table", routes.CustomTableID)
+		cmdArgs = append(ipBase, "route", "add", nextHop.String(), "dev", tunnelName, "table",
+			strconv.Itoa(routes.CustomTableID))
 		if out, err = exec.Command("ip", cmdArgs...).CombinedOutput(); err != nil {
 			return nil, fmt.Errorf("failed to add route in custom route table, err: %s, output: %s", err, string(out))
 		}

pkg/utils/ip.go

@@ -0,0 +1,117 @@
+package utils
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+)
+
+const (
+	IPv4DefaultRoute = "0.0.0.0/0"
+	IPv6DefaultRoute = "::/0"
+
+	ipv4NetMaskBits = 32
+	ipv6NetMaskBits = 128
+)
+
+// GetSingleIPNet returns an IPNet object that represents a subnet containing a single IP address for a given IP address
+// with proper handling for IPv4 and IPv6 addresses.
+func GetSingleIPNet(ip net.IP) *net.IPNet {
+	if ip.To4() != nil {
+		return &net.IPNet{
+			IP:   ip,
+			Mask: net.CIDRMask(ipv4NetMaskBits, ipv4NetMaskBits),
+		}
+	} else {
+		return &net.IPNet{
+			IP:   ip,
+			Mask: net.CIDRMask(ipv6NetMaskBits, ipv6NetMaskBits),
+		}
+	}
+}
+
+// GetIPv4NetMaxMaskBits returns the maximum mask bits for an IPv4 address
+func GetIPv4NetMaxMaskBits() uint32 {
+	return ipv4NetMaskBits
+}
+
+// GetIPv6NetMaxMaskBits returns the maximum mask bits for an IPv6 address
+func GetIPv6NetMaxMaskBits() uint32 {
+	return ipv6NetMaskBits
+}
+
+// ContainsIPv4Address checks a given string array to see if it contains a valid IPv4 address within it
+func ContainsIPv4Address(addrs []string) bool {
+	for _, addr := range addrs {
+		ip := net.ParseIP(addr)
+		if ip == nil {
+			continue
+		}
+		if ip.To4() != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// ContainsIPv6Address checks a given string array to see if it contains a valid IPv6 address within it
+func ContainsIPv6Address(addrs []string) bool {
+	for _, addr := range addrs {
+		ip := net.ParseIP(addr)
+		if ip == nil {
+			continue
+		}
+		if ip.To4() != nil {
+			continue
+		}
+		if ip.To16() != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// GetDefaultIPv4Route returns the default IPv4 route
+func GetDefaultIPv4Route() *net.IPNet {
+	_, defaultPrefixCIDR, err := net.ParseCIDR(IPv4DefaultRoute)
+	if err != nil {
+		return nil
+	}
+	return defaultPrefixCIDR
+}
+
+// GetDefaultIPv6Route returns the default IPv6 route
+func GetDefaultIPv6Route() *net.IPNet {
+	_, defaultPrefixCIDR, err := net.ParseCIDR(IPv6DefaultRoute)
+	if err != nil {
+		return nil
+	}
+	return defaultPrefixCIDR
+}
+
+// IPNetEqual checks if two IPNet objects are equal by comparing the IP and Mask
+func IPNetEqual(a, b *net.IPNet) bool {
+	if a == nil || b == nil {
+		return a == b
+	}
+	return a.IP.Equal(b.IP) && bytes.Equal(a.Mask, b.Mask)
+}
+
+// IsDefaultRoute checks if a given CIDR is a default route by comparing it to the default routes for IPv4 and IPv6
+func IsDefaultRoute(cidr *net.IPNet) (bool, error) {
+	var defaultPrefixCIDR *net.IPNet
+	var err error
+
+	if cidr.IP.To4() != nil {
+		_, defaultPrefixCIDR, err = net.ParseCIDR(IPv4DefaultRoute)
+		if err != nil {
+			return false, fmt.Errorf("failed to parse default route: %s", err.Error())
+		}
+	} else {
+		_, defaultPrefixCIDR, err = net.ParseCIDR(IPv6DefaultRoute)
+		if err != nil {
+			return false, fmt.Errorf("failed to parse default route: %s", err.Error())
+		}
+	}
+	return IPNetEqual(defaultPrefixCIDR, cidr), nil
+}