Skip to content

Commit

Permalink
Ponad and CNI plugin (#7)
Browse files Browse the repository at this point in the history
* change directory structure

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* fix Dockerfiles

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* add protobuf

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* [WIP] add ponad

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* [WIP] implementing ponad

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* [WIP] ponad Add

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* [WIP] implementing ponad

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* implemented collectDestinationsForEgress

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* add pkg/cni

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* [WIP] Add

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* [WIP] add nat client

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* [WIP] nc

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* [WIP] implement updateroutes

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* [WIP] UpdateRoutes

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* implemented nat client

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* implement Add

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* [WIP] add pona cni

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* add netip utility test

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* [WIP] implementing cni add

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* implement pona cmdAdd

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* implement pona & create pona-installer

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* [WIP] pona-installer

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* implement installer

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* use netns

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* [WIP] ponad debugging

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* fix netiputil

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* coil -> pona

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* add version

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* check-generate

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* update protoc-gen-go

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* change error

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* rm unused env

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* rm unused conf

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* add addthrow error

Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>

* fix throw route failure

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

* fix localip config

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>

---------

Signed-off-by: gotti <18141824+gotti@users.noreply.github.com>
Signed-off-by: walnuts1018 <r.juglans.1018@gmail.com>
Co-authored-by: gotti <18141824+gotti@users.noreply.github.com>
Co-authored-by: walnuts1018 <r.juglans.1018@gmail.com>
  • Loading branch information
3 people authored Aug 29, 2024
1 parent 90cd1ba commit adb5bea
Show file tree
Hide file tree
Showing 37 changed files with 2,356 additions and 191 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,6 @@ go.work
*.swp
*.swo
*~

# protobuf downloaded files
include
48 changes: 45 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,15 @@
IMG_TAG ?= dev
IMG_CONTROLLER ?= egress-controller:$(IMG_TAG)
IMG_GATEWAY ?= nat-gateway:$(IMG_TAG)
IMG_PONAD ?= ponad:$(IMG_TAG)
PONA_VERSION ?= dev-$(shell git rev-parse --short HEAD)

# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
ENVTEST_K8S_VERSION = 1.30.0

SUDO ?= sudo
PROTOC_OUTPUTS = pkg/cnirpc/cni.pb.go pkg/cnirpc/cni_grpc.pb.go docs/cni-grpc.md

# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))
GOBIN=$(shell go env GOPATH)/bin
Expand Down Expand Up @@ -52,6 +58,7 @@ manifests: controller-gen yq ## Generate WebhookConfiguration, ClusterRole and C

.PHONY: generate
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
$(MAKE) $(PROTOC_OUTPUTS)
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."

.PHONY: fmt
Expand All @@ -71,7 +78,9 @@ test: envtest manifests generate fmt vet mod ## Run tests.
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out

.PHONY: check-generate
check-generate: manifests generate fmt mod
check-generate: setup manifests fmt mod
-rm $(ROLES) $(PROTOC_OUTPUTS)
$(MAKE) generate
git diff --exit-code --name-only

# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
Expand Down Expand Up @@ -102,13 +111,21 @@ run: manifests generate fmt vet mod ## Run a controller from your host.
# More info: https://docs.docker.com/develop/develop-images/build_enhancements/
.PHONY: docker-build
docker-build: ## Build docker image with the manager.
$(CONTAINER_TOOL) build -t ${IMG_CONTROLLER} -f ./dockerfiles/Dockerfile.egress-controller .
$(CONTAINER_TOOL) build -t ${IMG_GATEWAY} -f ./dockerfiles/Dockerfile.nat-gateway .
$(CONTAINER_TOOL) build -t ${IMG_CONTROLLER} --build-arg PONA_VERSION=${PONA_VERSION} -f ./dockerfiles/Dockerfile.egress-controller .
$(CONTAINER_TOOL) build -t ${IMG_GATEWAY} --build-arg PONA_VERSION=${PONA_VERSION} -f ./dockerfiles/Dockerfile.nat-gateway .
$(CONTAINER_TOOL) build -t ${IMG_PONAD} --build-arg PONA_VERSION=${PONA_VERSION} -f ./dockerfiles/Dockerfile.ponad .

.PHONY: kind-load
kind-load:
kind load docker-image ${IMG_CONTROLLER}
kind load docker-image ${IMG_GATEWAY}
kind load docker-image ${IMG_PONAD}

.PHONY: docker-push
docker-push: ## Push docker image with the manager.
$(CONTAINER_TOOL) push ${IMG_CONTROLLER}
$(CONTAINER_TOOL) push ${IMG_GATEWAY}
$(CONTAINER_TOOL) push ${IMG_PONAD}

# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
Expand Down Expand Up @@ -172,13 +189,18 @@ GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
YQ = $(LOCALBIN)/yq
WGET_OPTIONS := --retry-on-http-error=503 --retry-connrefused --no-verbose
WGET = wget $(WGET_OPTIONS)
PROTOC := PATH=$(LOCALBIN):'$(PATH)' $(LOCALBIN)/protoc -I=$(PWD)/include:.

## Tool Versions
KUSTOMIZE_VERSION ?= v5.4.2
CONTROLLER_TOOLS_VERSION ?= v0.15.0
ENVTEST_VERSION ?= release-0.18
GOLANGCI_LINT_VERSION ?= v1.59.1
YQ_VERSION ?= 4.44.3
PROTOC_VERSION=27.3
PROTOC_GEN_GO_VERSION := $(shell awk '/google.golang.org\/protobuf/ {print substr($$2, 2)}' go.mod)
PROTOC_GEN_GO_GRPC_VERSON=1.5.1
PROTOC_GEN_DOC_VERSION=1.5.1

.PHONY: kustomize
kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
Expand All @@ -200,6 +222,26 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
$(GOLANGCI_LINT): $(LOCALBIN)
$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION))

.PHONY: setup
setup:
$(SUDO) apt-get update
$(SUDO) apt-get -y install --no-install-recommends unzip

curl -sfL -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-linux-x86_64.zip
unzip -o protoc.zip bin/protoc 'include/*'
rm -f protoc.zip
go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(PROTOC_GEN_GO_VERSION)
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v$(PROTOC_GEN_GO_GRPC_VERSON)
go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@v$(PROTOC_GEN_DOC_VERSION)

pkg/cnirpc/cni.pb.go: pkg/cnirpc/cni.proto
$(PROTOC) --go_out=module=github.com/cybozu-go/pona:. $<

pkg/cnirpc/cni_grpc.pb.go: pkg/cnirpc/cni.proto
$(PROTOC) --go-grpc_out=module=github.com/cybozu-go/pona:. $<

docs/cni-grpc.md: pkg/cnirpc/cni.proto
$(PROTOC) --doc_out=docs --doc_opt=markdown,$@ $<

.PHONY: yq
yq: $(YQ)
Expand Down
6 changes: 3 additions & 3 deletions cmd/nat-gateway/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ import (

ponav1beta1 "github.com/cybozu-go/pona/api/v1beta1"
"github.com/cybozu-go/pona/internal/controller"
"github.com/cybozu-go/pona/internal/nat"
"github.com/cybozu-go/pona/internal/tunnel/fou"
"github.com/cybozu-go/pona/pkg/nat"
"github.com/cybozu-go/pona/pkg/tunnel/fou"
// +kubebuilder:scaffold:imports
)

Expand Down Expand Up @@ -185,7 +185,7 @@ func main() {
setupLog.Error(err, "failed to Initialize FoUTunnelController")
os.Exit(1)
}
nc, err := nat.NewController("eth0", ipv4, ipv6)
nc, err := nat.NewGateway("eth0", ipv4, ipv6)
if err != nil {
setupLog.Error(err, "unable to create nat.Controller")
os.Exit(1)
Expand Down
44 changes: 44 additions & 0 deletions cmd/pona-installer/install.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package main

import (
"fmt"
"io"
"os"
"path/filepath"
)

func installPona(ponaPath, cniBinDir string) error {
f, err := os.Open(ponaPath)
if err != nil {
return fmt.Errorf("failed to read pona %w", err)
}
if err := os.MkdirAll(cniBinDir, 0755); err != nil {
return fmt.Errorf("failed to MkdirAll: %w", err)
}

g, err := os.CreateTemp(cniBinDir, ".tmp")
if err != nil {
return fmt.Errorf("failed to CreateTemp: %w", err)
}
defer func() {
g.Close()
os.Remove(g.Name())
}()

if _, err := io.Copy(g, f); err != nil {
return fmt.Errorf("failed to io.Copy: %w", err)
}

if err := g.Chmod(0755); err != nil {
return fmt.Errorf("failed to chmod: %w", err)
}

if err := g.Sync(); err != nil {
return fmt.Errorf("failed to Sync: %w", err)
}

if err := os.Rename(g.Name(), filepath.Join(cniBinDir, "pona")); err != nil {
return fmt.Errorf("failed to rename: %w", err)
}
return nil
}
29 changes: 29 additions & 0 deletions cmd/pona-installer/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package main

import (
"log/slog"
"os"

"github.com/caarlos0/env/v10"
_ "github.com/joho/godotenv/autoload"
)

type Config struct {
CniEtcDir string `env:"CNI_ETC_DIR" envDefault:"/host/etc/cni/net.d"`
CniBinDir string `env:"CNI_BIN_DIR" envDefault:"/host/opt/cni/bin"`
PonaPath string `env:"CNI_PATH" envDefault:"/pona"`
}

func main() {
var cfg Config
if err := env.Parse(&cfg); err != nil {
slog.Error("failed to parse config", slog.Any("error", err))
os.Exit(1)
}

if err := installPona(cfg.PonaPath, cfg.CniBinDir); err != nil {
slog.Error("failed to install pona",
slog.Any("error", err),
)
}
}
64 changes: 64 additions & 0 deletions cmd/pona/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package main

import (
"context"
"fmt"
"time"

"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
cni100 "github.com/containernetworking/cni/pkg/types/100"

"github.com/containernetworking/cni/pkg/version"
"github.com/cybozu-go/pona"
"github.com/cybozu-go/pona/pkg/cni"
"github.com/cybozu-go/pona/pkg/cnirpc"
)

func cmdAdd(args *skel.CmdArgs) error {
conf, err := cni.ParseConfig(args.StdinData)
if err != nil {
return types.NewError(types.ErrDecodingFailure, "failed to parse config from stdin data", err.Error())
}
if conf.PrevResult == nil {
return types.NewError(types.ErrInternal, "ponad must be called as chained plugin", "")
}

cniArgs, err := makeCNIArgs(args)
if err != nil {
return types.NewError(types.ErrInvalidNetworkConfig, "failed to transform args to RPC arg", err.Error())
}

conn, err := connect(conf.Socket)
if err != nil {
return types.NewError(types.ErrTryAgainLater, "failed to connect to socket", err.Error())
}

client := cnirpc.NewCNIClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()

resp, err := client.Add(ctx, cniArgs)
if err != nil {
return convertError(err)
}

result, err := cni100.NewResult(resp.Result)
if err != nil {
return types.NewError(types.ErrDecodingFailure, "failed to unmarshal result", err.Error())
}

return types.PrintResult(result, conf.CNIVersion)
}

func cmdDel(args *skel.CmdArgs) error {
return nil
}

func cmdCheck(args *skel.CmdArgs) error {
return nil
}

func main() {
skel.PluginMainFuncs(skel.CNIFuncs{Add: cmdAdd, Del: cmdDel, Check: cmdCheck, GC: nil, Status: nil}, version.PluginSupports("0.3.1", "0.4.0", "1.0.0", "1.1.0"), fmt.Sprintf("pona %s", pona.Version))
}
78 changes: 78 additions & 0 deletions cmd/pona/rpc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package main

import (
"context"
"fmt"
"net"

"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
"github.com/cybozu-go/pona/internal/constants"
"github.com/cybozu-go/pona/pkg/cnirpc"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/resolver"
"google.golang.org/grpc/status"
)

// PluginEnvArgs represents CNI_ARG
type PluginEnvArgs struct {
types.CommonArgs
K8S_POD_NAMESPACE types.UnmarshallableString
K8S_POD_NAME types.UnmarshallableString
K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString
}

// Map returns a map[string]string
func (e PluginEnvArgs) Map() map[string]string {
return map[string]string{
constants.PodNamespaceKey: string(e.K8S_POD_NAMESPACE),
constants.PodNameKey: string(e.K8S_POD_NAME),
constants.PodContainerKey: string(e.K8S_POD_INFRA_CONTAINER_ID),
}
}

func makeCNIArgs(args *skel.CmdArgs) (*cnirpc.CNIArgs, error) {
a := &PluginEnvArgs{}
if err := types.LoadArgs(args.Args, a); err != nil {
return nil, fmt.Errorf("failed to load args: %w", err)
}
return &cnirpc.CNIArgs{
ContainerId: args.ContainerID,
Netns: args.Netns,
Ifname: args.IfName,
Args: a.Map(),
Path: args.Path,
StdinData: args.StdinData,
}, nil
}

func connect(sockPath string) (*grpc.ClientConn, error) {
dialer := &net.Dialer{}
dialFunc := func(ctx context.Context, a string) (net.Conn, error) {
return dialer.DialContext(ctx, "unix", a)
}
resolver.SetDefaultScheme("passthrough")

conn, err := grpc.NewClient(sockPath, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialFunc))
if err != nil {
return nil, fmt.Errorf("failed to connect to %s: %w", sockPath, err)
}
return conn, nil
}

// convertError turns err returned from gRPC library into CNI's types.Error
func convertError(err error) error {
st := status.Convert(err)
details := st.Details()
if len(details) != 1 {
return types.NewError(types.ErrInternal, st.Message(), err.Error())
}

cniErr, ok := details[0].(*cnirpc.CNIError)
if !ok {
types.NewError(types.ErrInternal, st.Message(), err.Error())
}

return types.NewError(uint(cniErr.Code), cniErr.Msg, cniErr.Details)
}
Loading

0 comments on commit adb5bea

Please sign in to comment.