fix(Judger): 🚑 fix docker in docker's cgroup and fuse device isolation

Cargo.toml

@@ -11,6 +11,7 @@ members = [
 ]
 
 [workspace.dependencies]
+tikv-jemallocator = "0.5"
 prost = "0.12.3"
 prost-types = "0.12.3"
 toml = "0.7.4"

backend/Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 codegen-backend = "cranelift"
 
 [dependencies]
-tikv-jemallocator = { version = "0.5", optional = true }
+tikv-jemallocator = { workspace = true, optional = true }
 log = "0.4.18"
 paste = "1.0.12"
 toml = { workspace = true }

docker/dev/docker-compose.yml

@@ -32,11 +32,12 @@ services:
     privileged: true
     image: ghcr.io/mdcpp/mdoj/judger:staging
     profiles: [backend-dev, frontend-dev]
+    cgroup: host
+    devices:
+      - /dev/fuse:/dev/fuse 
     volumes:
       - ./judger/config:/config
       - ./judger/plugins:/plugins
-      - /sys/fs/cgroup:/sys/fs/cgroup
-      - /dev/fuse:/dev/fuse
     environment:
       - RUST_BACKTRACE=full
       - CONFIG_PATH=/config/config.toml

judger/Cargo.toml

@@ -9,6 +9,7 @@ edition = "2021"
 cgroups-rs = "0.3.4"
 env_logger = "0.10.1"
 futures-core = "0.3.30"
+tikv-jemallocator = { workspace = true, optional = true }
 prost = { workspace = true }
 prost-types = { workspace = true }
 thiserror = "1.0.40"

judger/Dockerfile

@@ -19,13 +19,15 @@ COPY . .
 RUN --mount=type=cache,target=target
 
 RUN rustup target add ${ARCH}-unknown-linux-musl
-RUN cargo install --target ${ARCH}-unknown-linux-musl --path judger
+RUN cargo install --profile dev --target ${ARCH}-unknown-linux-musl --path judger
 
-FROM scratch
+FROM alpine:3.20
 WORKDIR /plugins
 WORKDIR /config
 WORKDIR /
 
+RUN apk add --no-cache fuse3
+
 COPY --from=builder /usr/local/cargo/bin/judger /
 
 COPY judger/nsjail-3.1 /

judger/src/filesystem/adapter/fuse.rs

@@ -55,11 +55,6 @@ where
 
         mount_options.uid(uid).gid(gid).force_readdir_plus(true);
 
-        // FIXME: this panic in container
-        //
-        // additionally, libfuse report: `find fusermount3 binary failed`
-        metadata(path.as_ref()).await.expect("calling libc::mkdtemp actually creates the directory on host");
-
         Session::new(mount_options)
             .mount_with_unprivileged(self, path.as_ref())
             .await

judger/src/main.rs

@@ -10,6 +10,10 @@ pub use config::CONFIG;
 use grpc::judger::judger_server::JudgerServer;
 use server::Server;
 
+#[cfg(not(debug_assertions))]
+#[global_allocator]
+static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
 type Result<T> = std::result::Result<T, error::Error>;
 
 #[tokio::main]

judger/src/sandbox/monitor/mem_cpu.rs

@@ -111,8 +111,6 @@ impl super::Monitor for Monitor {
     /// This method is cancellation safe
     async fn wait_exhaust(&mut self) -> MonitorKind {
         let reason = self.monitor_task.as_mut().unwrap().await.unwrap();
-        // optimistic kill(`SIGKILL`) the process inside
-        self.cgroup.kill().expect("cgroup.kill does not exist");
         reason
     }
     fn poll_exhaust(&mut self) -> Option<MonitorKind> {

judger/src/sandbox/process/nsjail.rs

@@ -37,7 +37,8 @@ pub struct BaseArg;
 impl Argument for BaseArg {
     fn get_args(self) -> impl Iterator<Item = Cow<'static, OsStr>> {
         let mut args = vec![
-            Cow::Borrowed(OsStr::from_bytes(b"-Me")),
+            // FIXME: MODE_STANDALONE_ONCE would might cause sandbox to continue running after process exit, check if that's true
+            Cow::Borrowed(OsStr::from_bytes(b"-Mo")),
             Cow::Borrowed(OsStr::from_bytes(b"-l")),
             #[cfg(not(debug_assertions))]
             Cow::Borrowed(OsStr::from_bytes(b"/dev/null")),
@@ -73,8 +74,6 @@ impl<'a> Argument for CGroupMountArg<'a> {
                 Cow::Borrowed(OsStr::from_bytes(b"0")),
                 Cow::Borrowed(OsStr::from_bytes(b"--cgroup_cpu_parent")),
                 Cow::Owned(OsString::from(self.cg_name)),
-                // Cow::Borrowed(OsStr::from_bytes(b"--cgroupv2_mount")),
-                // Cow::Owned(OsString::from(self.cg_name)),
             ],
             false => vec![
                 Cow::Borrowed(OsStr::from_bytes(b"--disable_clone_newcgroup")),