Ponad and CNI plugin

.gitignore

@@ -25,3 +25,6 @@ go.work
 *.swp
 *.swo
 *~
+
+# protobuf downloaded files
+include

Makefile

@@ -2,9 +2,15 @@
 IMG_TAG ?= dev
 IMG_CONTROLLER ?= egress-controller:$(IMG_TAG)
 IMG_GATEWAY ?= nat-gateway:$(IMG_TAG)
+IMG_PONAD ?= ponad:$(IMG_TAG)
+PONA_VERSION ?= dev-$(shell git rev-parse --short HEAD)
+
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.30.0
 
+SUDO ?= sudo
+PROTOC_OUTPUTS = pkg/cnirpc/cni.pb.go pkg/cnirpc/cni_grpc.pb.go docs/cni-grpc.md
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -52,6 +58,7 @@ manifests: controller-gen yq ## Generate WebhookConfiguration, ClusterRole and C
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+	$(MAKE) $(PROTOC_OUTPUTS)
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
 .PHONY: fmt
@@ -71,7 +78,9 @@ test: envtest manifests generate fmt vet mod ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
 .PHONY: check-generate
-check-generate: manifests generate fmt mod
+check-generate: manifests fmt mod
+	-rm $(ROLES) $(PROTOC_OUTPUTS)
+	$(MAKE) generate
 	git diff --exit-code --name-only
 
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
@@ -102,13 +111,21 @@ run: manifests generate fmt vet mod ## Run a controller from your host.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager.
-	$(CONTAINER_TOOL) build -t ${IMG_CONTROLLER} -f ./dockerfiles/Dockerfile.egress-controller .
-	$(CONTAINER_TOOL) build -t ${IMG_GATEWAY} -f ./dockerfiles/Dockerfile.nat-gateway .
+	$(CONTAINER_TOOL) build -t ${IMG_CONTROLLER} --build-arg PONA_VERSION=${PONA_VERSION} -f ./dockerfiles/Dockerfile.egress-controller .
+	$(CONTAINER_TOOL) build -t ${IMG_GATEWAY} --build-arg PONA_VERSION=${PONA_VERSION} -f ./dockerfiles/Dockerfile.nat-gateway .
+	$(CONTAINER_TOOL) build -t ${IMG_PONAD} --build-arg PONA_VERSION=${PONA_VERSION} -f ./dockerfiles/Dockerfile.ponad .
+
+.PHONY: kind-load
+kind-load:
+	kind load docker-image ${IMG_CONTROLLER}
+	kind load docker-image ${IMG_GATEWAY}
+	kind load docker-image ${IMG_PONAD}
 
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
 	$(CONTAINER_TOOL) push ${IMG_CONTROLLER}
 	$(CONTAINER_TOOL) push ${IMG_GATEWAY}
+	$(CONTAINER_TOOL) push ${IMG_PONAD}
 
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
 # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
@@ -172,13 +189,18 @@ GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
 YQ = $(LOCALBIN)/yq
 WGET_OPTIONS := --retry-on-http-error=503 --retry-connrefused --no-verbose
 WGET = wget $(WGET_OPTIONS)
+PROTOC := PATH=$(LOCALBIN):'$(PATH)' $(LOCALBIN)/protoc -I=$(PWD)/include:.
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.4.2
 CONTROLLER_TOOLS_VERSION ?= v0.15.0
 ENVTEST_VERSION ?= release-0.18
 GOLANGCI_LINT_VERSION ?= v1.59.1
 YQ_VERSION ?= 4.44.3
+PROTOC_VERSION=27.3
+PROTOC_GEN_GO_VERSION := $(shell awk '/google.golang.org\/protobuf/ {print substr($$2, 2)}' go.mod)
+PROTOC_GEN_GO_GRPC_VERSON=1.5.1
+PROTOC_GEN_DOC_VERSION=1.5.1
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
@@ -200,6 +222,26 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
 	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION))
 
+.PHONY: setup
+setup:
+	$(SUDO) apt-get update
+	$(SUDO) apt-get -y install --no-install-recommends unzip
+
+	curl -sfL -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-linux-x86_64.zip
+	unzip -o protoc.zip bin/protoc 'include/*'
+	rm -f protoc.zip
+	go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(PROTOC_GEN_GO_VERSION)
+	go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v$(PROTOC_GEN_GO_GRPC_VERSON)
+	go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@v$(PROTOC_GEN_DOC_VERSION)
+
+pkg/cnirpc/cni.pb.go: pkg/cnirpc/cni.proto
+	$(PROTOC) --go_out=module=github.com/cybozu-go/pona:. $<
+
+pkg/cnirpc/cni_grpc.pb.go: pkg/cnirpc/cni.proto
+	$(PROTOC) --go-grpc_out=module=github.com/cybozu-go/pona:. $<
+
+docs/cni-grpc.md: pkg/cnirpc/cni.proto
+	$(PROTOC) --doc_out=docs --doc_opt=markdown,$@ $<
 
 .PHONY: yq
 yq: $(YQ)

cmd/nat-gateway/main.go

@@ -24,8 +24,8 @@ import (
 
 	ponav1beta1 "github.com/cybozu-go/pona/api/v1beta1"
 	"github.com/cybozu-go/pona/internal/controller"
-	"github.com/cybozu-go/pona/internal/nat"
-	"github.com/cybozu-go/pona/internal/tunnel/fou"
+	"github.com/cybozu-go/pona/pkg/nat"
+	"github.com/cybozu-go/pona/pkg/tunnel/fou"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -185,7 +185,7 @@ func main() {
 		setupLog.Error(err, "failed to Initialize FoUTunnelController")
 		os.Exit(1)
 	}
-	nc, err := nat.NewController("eth0", ipv4, ipv6)
+	nc, err := nat.NewGateway("eth0", ipv4, ipv6)
 	if err != nil {
 		setupLog.Error(err, "unable to create nat.Controller")
 		os.Exit(1)

cmd/pona-installer/install.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func installPona(ponaPath, cniBinDir string) error {
+	f, err := os.Open(ponaPath)
+	if err != nil {
+		return fmt.Errorf("failed to read pona %w", err)
+	}
+	if err := os.MkdirAll(cniBinDir, 0755); err != nil {
+		return fmt.Errorf("failed to MkdirAll: %w", err)
+	}
+
+	g, err := os.CreateTemp(cniBinDir, ".tmp")
+	if err != nil {
+		return fmt.Errorf("failed to CreateTemp: %w", err)
+	}
+	defer func() {
+		g.Close()
+		os.Remove(g.Name())
+	}()
+
+	_, err = io.Copy(g, f)
+	if err != nil {
+		return fmt.Errorf("failed to io.Copy: %w", err)
+	}
+
+	err = g.Chmod(0755)
+	if err != nil {
+		return fmt.Errorf("failed to chmod: %w", err)
+	}
+
+	err = g.Sync()
+	if err != nil {
+		return fmt.Errorf("failed to Sync: %w", err)
+	}
+
+	if err := os.Rename(g.Name(), filepath.Join(cniBinDir, "pona")); err != nil {
+		return fmt.Errorf("failed to rename: %w", err)
+	}
+	return nil
+}

cmd/pona-installer/main.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"log/slog"
+	"os"
+
+	"github.com/caarlos0/env/v10"
+	_ "github.com/joho/godotenv/autoload"
+)
+
+type Config struct {
+	CniEtcDir string `env:"CNI_ETC_DIR" envDefault:"/host/etc/cni/net.d"`
+	CniBinDir string `env:"CNI_BIN_DIR" envDefault:"/host/opt/cni/bin"`
+	PonaPath  string `env:"CNI_PATH" envDefault:"/pona"`
+	// CniNetConf  string `env:"CNI_NET_CONF"`
+	// CniConfName string `env:"CNI_CONF_NAME"`
+}
+
+func main() {
+	var cfg Config
+	if err := env.Parse(&cfg); err != nil {
+		slog.Error("failed to parse config", slog.Any("error", err))
+		os.Exit(1)
+	}
+
+	if err := installPona(cfg.PonaPath, cfg.CniBinDir); err != nil {
+		slog.Error("failed to install pona",
+			slog.Any("error", err),
+		)
+	}
+}

cmd/pona/main.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/types"
+	cni100 "github.com/containernetworking/cni/pkg/types/100"
+
+	"github.com/containernetworking/cni/pkg/version"
+	"github.com/cybozu-go/pona"
+	"github.com/cybozu-go/pona/pkg/cni"
+	"github.com/cybozu-go/pona/pkg/cnirpc"
+)
+
+func cmdAdd(args *skel.CmdArgs) error {
+	conf, err := cni.ParseConfig(args.StdinData)
+	if err != nil {
+		return types.NewError(types.ErrDecodingFailure, "failed to parse config from stdin data", err.Error())
+	}
+	if conf.PrevResult == nil {
+		return types.NewError(types.ErrInternal, "ponad must be called as chained plugin", "")
+	}
+
+	cniArgs, err := makeCNIArgs(args)
+	if err != nil {
+		return types.NewError(types.ErrInvalidNetworkConfig, "failed to transform args to RPC arg", err.Error())
+	}
+
+	conn, err := connect(conf.Socket)
+	if err != nil {
+		return types.NewError(types.ErrTryAgainLater, "failed to connect to socket", err.Error())
+	}
+
+	client := cnirpc.NewCNIClient(conn)
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	defer cancel()
+
+	resp, err := client.Add(ctx, cniArgs)
+	if err != nil {
+		return convertError(err)
+	}
+
+	result, err := cni100.NewResult(resp.Result)
+	if err != nil {
+		return types.NewError(types.ErrDecodingFailure, "failed to unmarshal result", err.Error())
+	}
+
+	return types.PrintResult(result, conf.CNIVersion)
+}
+
+func cmdDel(args *skel.CmdArgs) error {
+	return nil
+}
+
+func cmdCheck(args *skel.CmdArgs) error {
+	return nil
+}
+
+func main() {
+	skel.PluginMainFuncs(skel.CNIFuncs{Add: cmdAdd, Del: cmdDel, Check: cmdCheck, GC: nil, Status: nil}, version.PluginSupports("0.3.1", "0.4.0", "1.0.0", "1.1.0"), fmt.Sprintf("pona %s", pona.Version))
+}

cmd/pona/rpc.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/types"
+	"github.com/cybozu-go/pona/internal/constants"
+	"github.com/cybozu-go/pona/pkg/cnirpc"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+)
+
+// PluginEnvArgs represents CNI_ARG
+type PluginEnvArgs struct {
+	types.CommonArgs
+	K8S_POD_NAMESPACE          types.UnmarshallableString
+	K8S_POD_NAME               types.UnmarshallableString
+	K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString
+}
+
+// Map returns a map[string]string
+func (e PluginEnvArgs) Map() map[string]string {
+	return map[string]string{
+		constants.PodNamespaceKey: string(e.K8S_POD_NAMESPACE),
+		constants.PodNameKey:      string(e.K8S_POD_NAME),
+		constants.PodContainerKey: string(e.K8S_POD_INFRA_CONTAINER_ID),
+	}
+}
+
+func makeCNIArgs(args *skel.CmdArgs) (*cnirpc.CNIArgs, error) {
+	a := &PluginEnvArgs{}
+	if err := types.LoadArgs(args.Args, a); err != nil {
+		return nil, fmt.Errorf("failed to load args: %w", err)
+	}
+	return &cnirpc.CNIArgs{
+		ContainerId: args.ContainerID,
+		Netns:       args.Netns,
+		Ifname:      args.IfName,
+		Args:        a.Map(),
+		Path:        args.Path,
+		StdinData:   args.StdinData,
+	}, nil
+}
+
+func connect(sockPath string) (*grpc.ClientConn, error) {
+	dialer := &net.Dialer{}
+	dialFunc := func(ctx context.Context, a string) (net.Conn, error) {
+		return dialer.DialContext(ctx, "unix", a)
+	}
+	resolver.SetDefaultScheme("passthrough")
+
+	conn, err := grpc.NewClient(sockPath, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialFunc))
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to %s: %w", sockPath, err)
+	}
+	return conn, nil
+}
+
+// convertError turns err returned from gRPC library into CNI's types.Error
+func convertError(err error) error {
+	st := status.Convert(err)
+	details := st.Details()
+	if len(details) != 1 {
+		return types.NewError(types.ErrInternal, st.Message(), err.Error())
+	}
+
+	cniErr, ok := details[0].(*cnirpc.CNIError)
+	if !ok {
+		types.NewError(types.ErrInternal, st.Message(), err.Error())
+	}
+
+	return types.NewError(uint(cniErr.Code), cniErr.Msg, cniErr.Details)
+}