envexec: add optional flag for copyOut files

fix #14

README.cn.md

@@ -213,6 +213,7 @@ interface Cmd {
     copyIn?: {[dst:string]:LocalFile | MemoryFile | PreparedFile};
 
     // 在执行程序后从容器文件系统中复制出来的文件列表
+    // 在文件名之后加入 '?' 来使文件变为可选，可选文件不存在的情况不会触发 FileError
     copyOut?: string[];
     // 和 copyOut 相同，不过文件不返回内容，而是返回一个对应文件 ID ，内容可以通过 /file/:fileId 接口下载
     copyOutCached?: string[];

README.md

@@ -256,6 +256,7 @@ interface Cmd {
     copyIn?: {[dst:string]:LocalFile | MemoryFile | PreparedFile};
 
     // copy out specifies files need to be copied out from the container after execution
+    // append '?' after file name will make the file optional and do not cause FileError when missing
     copyOut?: string[];
     // similar to copyOut but stores file in executor service and returns fileId, later download through /file/:fileId
     copyOutCached?: string[];

cmd/executorserver/grpc_executor/grpc.go

@@ -52,7 +52,7 @@ func (e *execServer) Exec(ctx context.Context, req *pb.Request) (*pb.Response, e
 		return nil, err
 	}
 	if len(si) > 0 || len(so) > 0 {
-		return nil, fmt.Errorf("Stream in / out are not available for exec request")
+		return nil, fmt.Errorf("stream in / out are not available for exec request")
 	}
 	e.logger.Sugar().Debugf("request: %+v", r)
 	rt := <-e.worker.Submit(ctx, r)
@@ -203,8 +203,8 @@ func convertPBCmd(c *pb.Request_CmdType, srcPrefix string) (cm worker.Cmd, strea
 		ProcLimit:         c.GetProcLimit(),
 		CPURateLimit:      c.GetCPURateLimit(),
 		StrictMemoryLimit: c.GetStrictMemoryLimit(),
-		CopyOut:           c.GetCopyOut(),
-		CopyOutCached:     c.GetCopyOutCached(),
+		CopyOut:           convertCopyOut(c.GetCopyOut()),
+		CopyOutCached:     convertCopyOut(c.GetCopyOutCached()),
 		CopyOutMax:        c.GetCopyOutMax(),
 		CopyOutDir:        c.GetCopyOutDir(),
 	}
@@ -277,3 +277,14 @@ func checkPathPrefix(path, prefix string) (bool, error) {
 	}
 	return strings.HasPrefix(filepath.Join(wd, path), prefix), nil
 }
+
+func convertCopyOut(copyOut []*pb.Request_CmdCopyOutFile) []envexec.CmdCopyOutFile {
+	rt := make([]envexec.CmdCopyOutFile, 0, len(copyOut))
+	for _, n := range copyOut {
+		rt = append(rt, envexec.CmdCopyOutFile{
+			Name:     n.GetName(),
+			Optional: n.GetOptional(),
+		})
+	}
+	return rt
+}

cmd/executorserver/main.go

@@ -341,9 +341,10 @@ func newWorker(conf *config.Config, envPool worker.EnvironmentPool, fs filestore
 
 func handleVersion(c *gin.Context) {
 	c.JSON(http.StatusOK, gin.H{
-		"buildVersion": version.Version,
-		"goVersion":    runtime.Version(),
-		"platform":     runtime.GOARCH,
-		"os":           runtime.GOOS,
+		"buildVersion":    version.Version,
+		"goVersion":       runtime.Version(),
+		"platform":        runtime.GOARCH,
+		"os":              runtime.GOOS,
+		"copyOutOptional": true,
 	})
 }

cmd/executorserver/model/model.go

@@ -185,8 +185,8 @@ func convertCmd(c Cmd, srcPrefix string) (worker.Cmd, error) {
 		ProcLimit:         c.ProcLimit,
 		CPURateLimit:      c.CPURateLimit,
 		StrictMemoryLimit: c.StrictMemoryLimit,
-		CopyOut:           c.CopyOut,
-		CopyOutCached:     c.CopyOutCached,
+		CopyOut:           convertCopyOut(c.CopyOut),
+		CopyOutCached:     convertCopyOut(c.CopyOutCached),
 		CopyOutMax:        c.CopyOutMax,
 		CopyOutDir:        c.CopyOutDir,
 	}
@@ -246,3 +246,22 @@ func checkPathPrefix(path, prefix string) (bool, error) {
 	}
 	return strings.HasPrefix(filepath.Join(wd, path), prefix), nil
 }
+
+const optionalSuffix = "?"
+
+func convertCopyOut(copyOut []string) []envexec.CmdCopyOutFile {
+	rt := make([]envexec.CmdCopyOutFile, 0, len(copyOut))
+	for _, n := range copyOut {
+		if strings.HasSuffix(n, optionalSuffix) {
+			rt = append(rt, envexec.CmdCopyOutFile{
+				Name:     strings.TrimSuffix(n, optionalSuffix),
+				Optional: true,
+			})
+			continue
+		}
+		rt = append(rt, envexec.CmdCopyOutFile{
+			Name: n,
+		})
+	}
+	return rt
+}

env/linuxcontainer/environment_linux.go

@@ -107,7 +107,7 @@ func (c *environ) WorkDir() *os.File {
 func (c *environ) Open(path string, flags int, perm os.FileMode) (*os.File, error) {
 	fd, err := syscall.Openat(int(c.wd.Fd()), path, flags|syscall.O_CLOEXEC, uint32(perm))
 	if err != nil {
-		return nil, fmt.Errorf("openAtWorkDir: %v", err)
+		return nil, &os.PathError{Op: "open", Path: path, Err: err}
 	}
 	f := os.NewFile(uintptr(fd), path)
 	if f == nil {

envexec/cmd.go

@@ -44,13 +44,19 @@ type Cmd struct {
 	Waiter func(context.Context, Process) bool
 
 	// file names to copyout after exec
-	CopyOut    []string
+	CopyOut    []CmdCopyOutFile
 	CopyOutMax Size // file size limit
 
 	// CopyOutDir specifies a dir to dump all /w contnet
 	CopyOutDir string
 }
 
+// CmdCopyOutFile defines the file to be copy out after cmd execution
+type CmdCopyOutFile struct {
+	Name     string // Name is the file out to copyOut
+	Optional bool   // Optional ignores the file if not exists
+}
+
 // Result defines the running result for single Cmd
 type Result struct {
 	Status Status

envexec/file_collect.go

@@ -2,6 +2,7 @@ package envexec
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -28,8 +29,11 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string][
 	for _, n := range c.CopyOut {
 		n := n
 		g.Go(func() error {
-			cf, err := m.Open(n, os.O_RDONLY, 0777)
+			cf, err := m.Open(n.Name, os.O_RDONLY, 0777)
 			if err != nil {
+				if errors.Is(err, os.ErrNotExist) && n.Optional {
+					return nil
+				}
 				return err
 			}
 			defer cf.Close()
@@ -40,12 +44,12 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string][
 			}
 			// check regular file
 			if stat.Mode()&os.ModeType != 0 {
-				return fmt.Errorf("File(%s) is not a regular file %d", n, stat.Mode()&os.ModeType)
+				return fmt.Errorf("%s: not a regular file %d", n.Name, stat.Mode()&os.ModeType)
 			}
 			// check size limit
 			s := stat.Size()
 			if c.CopyOutMax > 0 && s > int64(c.CopyOutMax) {
-				return fmt.Errorf("File(%s) have size (%d) exceeded the limit (%d)", n, s, c.CopyOutMax)
+				return fmt.Errorf("%s: size (%d) exceeded the limit (%d)", n.Name, s, c.CopyOutMax)
 			}
 			var buf bytes.Buffer
 			buf.Grow(int(s))
@@ -55,7 +59,7 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string][
 			if err != nil {
 				return err
 			}
-			put(buf.Bytes(), n)
+			put(buf.Bytes(), n.Name)
 			return nil
 		})
 	}