Merge pull request #92 from miscord-dev/use-netns

Use netns

.gitignore

@@ -16,3 +16,5 @@ bin/
 # vendor/
 tetrapod.yaml
 tetrapod.db
+
+tmp/

Makefile

@@ -34,7 +34,10 @@ hostvrf: bin
 route-pods: bin
 	CGO_ENABLED=0 go build -o ./bin ./tetracni/cmd/route-pods
 
-cni-plugins: tetra-extra-routes tetra-pod-ipam hostvrf route-pods
+nsexec: bin
+	CGO_ENABLED=0 go build -o ./bin ./tetracni/cmd/nsexec
+
+cni-plugins: tetra-extra-routes tetra-pod-ipam hostvrf route-pods nsexec
 
 .PHONY: test
 test: envtest ## Run tests.
@@ -44,3 +47,16 @@ test: envtest ## Run tests.
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
 	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+
+.PHONY: install-cnitools
+install-cnitools:
+	GOBIN=$(PWD)/bin go install github.com/containernetworking/cni/cnitool@latest
+	( \
+		rm -rf plugins; \
+		git clone https://github.com/containernetworking/plugins.git && \
+		cd plugins && \
+		./build_linux.sh && \
+		mv ./bin/* ../bin && \
+		cd .. && \
+		rm -rf plugins \
+	)

controlplane/config/default/kustomization.yaml

@@ -12,7 +12,7 @@ namePrefix: controlplane-
 #commonLabels:
 #  someName: someValue
 
-bases:
+resources:
 - ../crd
 - ../rbac
 - ../manager

pkg/nsutil/nsutil.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 	"runtime"
+	"sync"
 
 	"github.com/vishvananda/netns"
 )
@@ -38,31 +39,44 @@ func CreateNamespace(name string) (netns.NsHandle, error) {
 }
 
 func RunInNamespace(handle netns.NsHandle, fn func() error) (err error) {
-	runtime.LockOSThread()
-	defer func() {
-		if err == nil {
-			runtime.UnlockOSThread()
+	impl := func() error {
+		runtime.LockOSThread()
+		defer func() {
+			if err == nil {
+				runtime.UnlockOSThread()
+			}
+		}()
+
+		cur, err := netns.Get()
+		if err != nil {
+			return err
 		}
-	}()
+		defer func() {
+			if e := netns.Set(cur); e != nil {
+				err = fmt.Errorf("failed to recover netns: %w", err)
+			}
+			cur.Close()
+		}()
 
-	cur, err := netns.Get()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if e := netns.Set(cur); e != nil {
-			err = fmt.Errorf("failed to recover netns: %w", err)
+		if err := netns.Set(handle); err != nil {
+			return fmt.Errorf("failed to set netns: %w", err)
 		}
-		cur.Close()
-	}()
 
-	if err := netns.Set(handle); err != nil {
-		return fmt.Errorf("failed to set netns: %w", err)
-	}
+		if err := fn(); err != nil {
+			return fmt.Errorf("fn failed: %w", err)
+		}
 
-	if err := fn(); err != nil {
-		return fmt.Errorf("fn failed: %w", err)
+		return nil
 	}
 
-	return nil
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		err = impl()
+	}()
+
+	wg.Wait()
+
+	return err
 }

tetracni/cmd/nsexec/main.go

@@ -4,8 +4,12 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
+	"os"
 	"os/exec"
+	"path/filepath"
 
+	"github.com/containernetworking/cni/pkg/invoke"
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
 	"github.com/containernetworking/cni/pkg/version"
@@ -40,17 +44,35 @@ func cmd(args *skel.CmdArgs) error {
 		return err
 	}
 
+	_, _ = nsutil.CreateNamespace(netConf.Sandbox)
+
 	handle, err := netns.GetFromName(netConf.Sandbox)
 
 	if err != nil {
 		return err
 	}
 
+	paths := filepath.SplitList(os.Getenv("CNI_PATH"))
+	pluginPath, err := invoke.FindInPath(netConf.Plugin, paths)
+	if err != nil {
+		return err
+	}
+
 	return nsutil.RunInNamespace(handle, func() error {
-		cmd := exec.Command(netConf.Plugin)
+		cmd := exec.Command(pluginPath)
 
 		cmd.Stdin = bytes.NewReader(args.StdinData)
 
-		return cmd.Run()
+		var buf bytes.Buffer
+		cmd.Stdout = io.MultiWriter(os.Stdout, &buf)
+		cmd.Stderr = io.MultiWriter(os.Stderr, &buf)
+
+		err := cmd.Run()
+
+		if err != nil {
+			return fmt.Errorf("executing plugin failed %s: %w", buf.String(), err)
+		}
+
+		return nil
 	})
 }

tetracni/cmd/route-pods/conf.go

@@ -12,6 +12,7 @@ import (
 type Conf struct {
 	types.NetConf
 
+	Sandbox       string `json:"sandbox"`
 	HostVeth      string `json:"hostVeth"`
 	PeerVeth      string `json:"peerVeth"`
 	Firewall      string `json:"firewall"`

tetracni/cmd/route-pods/iptables.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 
 	"github.com/coreos/go-iptables/iptables"
+	"github.com/miscord-dev/tetrapod/pkg/nsutil"
 	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
 )
 
 const (
@@ -19,7 +21,7 @@ var (
 	}
 )
 
-func setUpIPTables(ipt *iptables.IPTables, hostVeth *netlink.Veth, redirectedChain string) error {
+func setUpIPTables(ipt *iptables.IPTables, peerVeth *netlink.Veth, redirectedChain string) error {
 	exists, err := ipt.ChainExists(iptablesFilterTable, redirectedChain)
 
 	if err != nil {
@@ -41,9 +43,9 @@ func setUpIPTables(ipt *iptables.IPTables, hostVeth *netlink.Veth, redirectedCha
 	}
 
 	rules := [][]string{
-		{"-o", hostVeth.Name, "-j", "ACCEPT"},
-		{"-i", hostVeth.Name, "-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT"},
-		{"-i", hostVeth.Name, "-j", "DROP"},
+		{"-i", peerVeth.Name, "-j", "ACCEPT"},
+		{"-o", peerVeth.Name, "-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT"},
+		{"-o", peerVeth.Name, "-j", "DROP"},
 	}
 
 	for _, rule := range rules {
@@ -55,29 +57,37 @@ func setUpIPTables(ipt *iptables.IPTables, hostVeth *netlink.Veth, redirectedCha
 	return nil
 }
 
-func setUpFirewall(hostVeth *netlink.Veth, conf *Conf) error {
+func setUpFirewall(peerNetns netns.NsHandle, peerVeth *netlink.Veth, conf *Conf) error {
 	switch conf.Firewall {
 	case FirewallNever:
 		return nil
 	case FirewallIPTables:
-		ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+		err := nsutil.RunInNamespace(peerNetns, func() error {
+			ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 
-		if err != nil {
-			return fmt.Errorf("failed to set up iptables: %w", err)
-		}
+			if err != nil {
+				return fmt.Errorf("failed to set up iptables: %w", err)
+			}
 
-		if err := setUpIPTables(ipt, hostVeth, conf.IPTablesChain); err != nil {
-			return fmt.Errorf("failed to set up iptables for v4: %w", err)
-		}
+			if err := setUpIPTables(ipt, peerVeth, conf.IPTablesChain); err != nil {
+				return fmt.Errorf("failed to set up iptables for v4: %w", err)
+			}
 
-		ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv6)
+			ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv6)
 
-		if err != nil {
-			return fmt.Errorf("failed to set up iptables: %w", err)
-		}
+			if err != nil {
+				return fmt.Errorf("failed to set up iptables: %w", err)
+			}
+
+			if err := setUpIPTables(ipt, peerVeth, conf.IPTablesChain); err != nil {
+				return fmt.Errorf("failed to set up iptables for v4: %w", err)
+			}
 
-		if err := setUpIPTables(ipt, hostVeth, conf.IPTablesChain); err != nil {
-			return fmt.Errorf("failed to set up iptables for v4: %w", err)
+			return nil
+		})
+
+		if err != nil {
+			return fmt.Errorf("manipulating iptables in %s netns failed: %w", peerNetns, err)
 		}
 	}