Mount cgroupv2 from temp dir (#411)

DEVELOPMENT.md

@@ -244,6 +244,18 @@ cat /sys/kernel/debug/tracing/available_filter_functions | grep socket_connect
 4. Click generate release notes.
 5. Publish release.
 
+## Testing runtime on real k8s cluster
+
+To install local kvisor chart to real cluster and test runtime you can run. In this mode it will only output ebpf events to stdout.
+
+```
+helm upgrade -i castai-kvisor ./charts/kvisor/ -n castai-agent --create-namespace \
+    --set image.tag=<your-pr-image-tag> \
+    --set castai.enabled=false \
+    --set agent.enabled=true \
+    --set agent.extraArgs.ebpf-events-enabled=true \
+    --set agent.extraArgs.ebpf-events-stdio-exporter-enabled=true
+```
 
 ## Testing netflow
 
@@ -386,3 +398,23 @@ kvisor                         69801            kvisor-agent     perf_event_open
 ```
 
 Any `Deny` in the `VERDICT` column should be added to the `containerSecurityContext` of the kvisor agent container in the `charts/kvisor/values.yaml` file to be allowed.
+
+
+## Creating EKS cluster
+
+```sh
+eksctl create cluster \                 
+  --name <your-cluster-name> \
+  --region us-east-1 \
+  --nodes 1 \
+  --nodes-min 1 \
+  --nodes-max 1 \
+  --node-type t3.medium \
+  --managed
+```
+
+To delete cluster
+
+```
+eksctl delete cluster <your-cluster-name> --region=us-east-1
+```

charts/kvisor/templates/agent.yaml

@@ -137,6 +137,8 @@ spec:
             - name: cgroups
               mountPath: /cgroups
               readOnly: true
+            - mountPath: /cgroup2-manual-mount
+              name: cgroup-mountdir
             - name: etc-os-release
               mountPath: /etc/os-release-host
               readOnly: true
@@ -172,6 +174,8 @@ spec:
         - name: cgroups
           hostPath:
             path: /sys/fs/cgroup
+        - name: cgroup-mountdir
+          emptyDir: {}
         - name: containerd-k8s-rootfs
           hostPath:
             path: /run/containerd/io.containerd.runtime.v2.task/k8s.io

cmd/agent/daemon/app/app.go

@@ -74,6 +74,7 @@ type Config struct {
 	Clickhouse                     ClickhouseConfig                `json:"clickhouse"`
 	KubeAPIServiceAddr             string                          `json:"kubeAPIServiceAddr"`
 	ExportersQueueSize             int                             `validate:"required" json:"exportersQueueSize"`
+	AutomountCgroupv2              bool                            `json:"automountCgroupv2"`
 }
 
 type EnricherConfig struct {
@@ -275,6 +276,7 @@ func (a *App) Run(ctx context.Context) error {
 		DefaultCgroupsVersion:              cgroupClient.DefaultCgroupVersion().String(),
 		ContainerClient:                    containersClient,
 		CgroupClient:                       cgroupClient,
+		AutomountCgroupv2:                  a.cfg.AutomountCgroupv2,
 		SignatureEngine:                    signatureEngine,
 		MountNamespacePIDStore:             mountNamespacePIDStore,
 		HomePIDNS:                          pidNSID,

cmd/agent/daemon/daemon.go

@@ -102,6 +102,8 @@ func NewRunCommand(version string) *cobra.Command {
 
 		exportersQueueSize = command.Flags().Int("exporters-queue-size", 4096, "Exporters queue size")
 
+		automountCgroupv2 = command.Flags().Bool("automount-cgroupv2", true, "Automount cgroupv2 if not mounted")
+
 		redactSensitiveValuesRegexStr = command.Flags().String("redact-sensitive-values-regex", "", "Regex which will be used to detect sensitive values in process exec args")
 	)
 
@@ -185,6 +187,7 @@ func NewRunCommand(version string) *cobra.Command {
 			},
 			KubeAPIServiceAddr: *kubeAPIServiceAddr,
 			ExportersQueueSize: *exportersQueueSize,
+			AutomountCgroupv2:  *automountCgroupv2,
 		}).Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
 			slog.Error(err.Error())
 			os.Exit(1)

pkg/ebpftracer/module.go

@@ -21,14 +21,9 @@ import (
 
 type TracerEventContextT = tracerEventContextT
 
-type moduleConfig struct {
-	BTFObjPath string
-}
-
-func newModule(log *logging.Logger, cfg moduleConfig) *module {
+func newModule(log *logging.Logger) *module {
 	return &module{
 		log:            log,
-		cfg:            cfg,
 		loaded:         &atomic.Bool{},
 		attachedProbes: map[handle]struct{}{},
 	}
@@ -38,7 +33,6 @@ func newModule(log *logging.Logger, cfg moduleConfig) *module {
 type module struct {
 	log     *logging.Logger
 	objects *tracerObjects
-	cfg     moduleConfig
 
 	networkTrafficSummaryMapSpec *ebpf.MapSpec
 
@@ -68,8 +62,8 @@ func (m *module) load(cfg Config) error {
 	}
 
 	var kernelTypes *btf.Spec
-	if m.cfg.BTFObjPath != "" {
-		kernelTypes, err = btf.LoadSpec(m.cfg.BTFObjPath)
+	if cfg.BTFPath != "" {
+		kernelTypes, err = btf.LoadSpec(cfg.BTFPath)
 		if err != nil {
 			return fmt.Errorf("loading custom btf: %w", err)
 		}
@@ -118,16 +112,17 @@ func (m *module) load(cfg Config) error {
 
 	m.objects = &objs
 
-	// TODO(Kvisord): Mount cgroupv2 if not mounted.
+	// Make sure cgroupv2 is mounted. It's required for cgroup networking ebpf programs.
 	cgroupPath, err := detectCgroupPath(cfg.CgroupClient.GetCgroupsRootPath())
-	if err != nil {
-		cgroupPath = "/cgroupv2"
-		m.log.Debugf("mounting cgroupv2 to path %s", cgroupPath)
+	if err != nil && cfg.AutomountCgroupv2 {
+		// Path /cgroup2-manual-mount is created as a temp dir from the host using volume mount.
+		cgroupPath = "/cgroup2-manual-mount/cgroupv2"
+		m.log.Infof("mounting cgroupv2 to path %s", cgroupPath)
 		if err := mountCgroup2(cgroupPath); err != nil {
 			return fmt.Errorf("mounting cgroupv2: %w", err)
 		}
 	}
-	m.log.Debugf("using cgroup path: %s", cgroupPath)
+	m.log.Infof("using cgroup path: %s", cgroupPath)
 	m.probes = newProbes(m.objects, cgroupPath)
 
 	m.loaded.Store(true)

pkg/ebpftracer/tracer.go

@@ -70,6 +70,7 @@ type Config struct {
 	EventsOutputChanSize   int
 	DefaultCgroupsVersion  string `validate:"required,oneof=V1 V2"`
 	DebugEnabled           bool
+	AutomountCgroupv2      bool
 	ContainerClient        ContainerClient
 	CgroupClient           CgroupClient
 	SignatureEngine        *signature.SignatureEngine
@@ -127,9 +128,7 @@ func New(log *logging.Logger, cfg Config) *Tracer {
 	}
 
 	log = log.WithField("component", "ebpftracer")
-	m := newModule(log, moduleConfig{
-		BTFObjPath: cfg.BTFPath,
-	})
+	m := newModule(log)
 
 	if cfg.EventsPerCPUBuffer == 0 {
 		cfg.EventsPerCPUBuffer = 8192