fix: timeout zombie processes in alpine based containers (#97)

cmd/containerhelper/handlers/all.go

@@ -19,4 +19,5 @@ func init() {
 	model.RegisterHandler("httpProxy", httpProxyHandler)
 	model.RegisterHandler("tcpProxy", tcpProxyHandler)
 	model.RegisterHandler("fixout", fixOutHandler)
+	model.RegisterHandler("countZombies", countZombiesHandler)
 }

cmd/containerhelper/handlers/countZombies.go

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
+ */
+
+package handlers
+
+import (
+	"errors"
+	"github.com/traas-stack/holoinsight-agent/cmd/containerhelper/model"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+const (
+	countLimit = 100
+)
+
+// Count the number of zombie processes
+func countZombiesHandler(_ string, resp *model.Resp) error {
+	count := 0
+	filepath.Walk("/proc", func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			return nil
+		}
+		if _, err := strconv.ParseInt(info.Name(), 10, 32); err != nil {
+			return nil
+		}
+		content, err := os.ReadFile(filepath.Join(path, "/status"))
+		if err != nil {
+			return nil
+		}
+		lines := strings.Split(string(content), "\n")
+		for _, line := range lines {
+			if strings.HasPrefix(line, "State:") {
+				if strings.Contains(line, "zombie") {
+					count++
+				}
+				break
+			}
+		}
+		if count > countLimit {
+			return errors.New("SkipAll")
+		}
+		return filepath.SkipDir
+	})
+	resp.Data = count
+	return nil
+}

cmd/containerhelper/handlers/countZombies.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -e
+
+# docs: count zombies processes
+COUNT_LIMIT=100
+
+count=0
+for i in /proc/*; do
+  b=`basename $i`
+  if [ -e "$i/status" ]; then
+    if grep 'State:	Z (zombie)' "$i/status" >/dev/null 2>&1; then
+      count=`expr $count + 1`
+      if [ "$count" -gt "$COUNT_LIMIT" ]; then
+        break
+      fi
+    fi
+  fi
+done
+
+echo $count

cmd/containerhelper/main.go

@@ -10,12 +10,17 @@ import (
 	_ "github.com/traas-stack/holoinsight-agent/cmd/containerhelper/handlers"
 	"github.com/traas-stack/holoinsight-agent/cmd/containerhelper/model"
 	"os"
+	"time"
 )
 
-// 这是一个辅助的bin, k8s场景下, 该bin会被复制到容器内, 然后 daemonset agent 会 使用 docker exec 调用该 bin 在容器内部执行一些命令(从而避免在daemonset agent 通过 切换namespace去访问)
-// 入参 通过 args 和 stdin(内容是一个json) 传输
-// 出参 通过 stdout(内容是一个json) 传输
+// This is a helper binary.
+// It will be copied to container.
+// It will be called by docker exec API to detect certain information inside the container.
 func main() {
+	// Under normal circumstances, the helper program should not be executed for a long time. If the time is exceeded, it will be forced to exit.
+	time.AfterFunc(3*time.Minute, func() {
+		os.Exit(97)
+	})
 	var resp = &model.Resp{}
 
 	defer func() {

pkg/core/helper.go

@@ -10,7 +10,9 @@ import (
 )
 
 var HelperToolLocalPath = "/usr/local/holoinsight/agent/bin/helper"
+var BusyboxLocalPath = "/usr/local/holoinsight/agent/bin/busybox"
 var HelperToolPath = "/tmp/holoinsight/helper"
+var BusyboxPath = "/tmp/holoinsight/busybox"
 
 type (
 	HelperBaseResp struct {

pkg/cri/criutils/zombies.go

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
+ */
+
+package criutils
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/spf13/cast"
+	"github.com/traas-stack/holoinsight-agent/cmd/containerhelper/model"
+	"github.com/traas-stack/holoinsight-agent/pkg/core"
+	"github.com/traas-stack/holoinsight-agent/pkg/cri"
+)
+
+func CountZombies(i cri.Interface, ctx context.Context, c *cri.Container) (int, error) {
+	r, err := i.Exec(ctx, c, cri.ExecRequest{
+		Cmd: []string{core.HelperToolPath, "countZombies"},
+	})
+	if err != nil {
+		return 0, err
+	}
+	resp := &model.Resp{}
+	if err := json.NewDecoder(r.Stdout).Decode(resp); err != nil {
+		return 0, err
+	}
+	return cast.ToIntE(resp.Data)
+}