From 9f7a0c3471bfcc5562c1473a97c788f0e8708e18 Mon Sep 17 00:00:00 2001 From: "Eason(G Ray)" <30045503+Eason0729@users.noreply.github.com> Date: Thu, 4 Jul 2024 13:07:36 +0800 Subject: [PATCH] fix(Judger): :ambulance: fix docker in docker's cgroup and fuse device isolation --- Cargo.lock | 1 + Cargo.toml | 1 + backend/Cargo.toml | 2 +- docker/dev/docker-compose.yml | 5 +++-- judger/Cargo.toml | 1 + judger/Dockerfile | 6 ++++-- judger/src/filesystem/adapter/fuse.rs | 5 ----- judger/src/main.rs | 4 ++++ judger/src/sandbox/monitor/mem_cpu.rs | 2 -- judger/src/sandbox/process/nsjail.rs | 5 ++--- 10 files changed, 17 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d13719..b667af8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2862,6 +2862,7 @@ dependencies = [ "tar", "tempfile", "thiserror", + "tikv-jemallocator", "tokio", "tokio-stream", "toml 0.7.8", diff --git a/Cargo.toml b/Cargo.toml index 11a7e08..b55c038 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ ] [workspace.dependencies] +tikv-jemallocator = "0.5" prost = "0.12.3" prost-types = "0.12.3" toml = "0.7.4" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index edfabc1..f3a2576 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" codegen-backend = "cranelift" [dependencies] -tikv-jemallocator = { version = "0.5", optional = true } +tikv-jemallocator = { workspace = true, optional = true } log = "0.4.18" paste = "1.0.12" toml = { workspace = true } diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index 9f3913f..aec1e95 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -32,11 +32,12 @@ services: privileged: true image: ghcr.io/mdcpp/mdoj/judger:staging profiles: [backend-dev, frontend-dev] + cgroup: host + devices: + - /dev/fuse:/dev/fuse volumes: - ./judger/config:/config - ./judger/plugins:/plugins - - /sys/fs/cgroup:/sys/fs/cgroup - - /dev/fuse:/dev/fuse environment: - RUST_BACKTRACE=full - CONFIG_PATH=/config/config.toml diff --git a/judger/Cargo.toml b/judger/Cargo.toml index 5faedc7..6bc1ffe 100644 --- a/judger/Cargo.toml +++ b/judger/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" cgroups-rs = "0.3.4" env_logger = "0.10.1" futures-core = "0.3.30" +tikv-jemallocator = { workspace = true, optional = true } prost = { workspace = true } prost-types = { workspace = true } thiserror = "1.0.40" diff --git a/judger/Dockerfile b/judger/Dockerfile index 2857a11..6b38dc1 100644 --- a/judger/Dockerfile +++ b/judger/Dockerfile @@ -19,13 +19,15 @@ COPY . . RUN --mount=type=cache,target=target RUN rustup target add ${ARCH}-unknown-linux-musl -RUN cargo install --target ${ARCH}-unknown-linux-musl --path judger +RUN cargo install --profile dev --target ${ARCH}-unknown-linux-musl --path judger -FROM scratch +FROM alpine:3.20 WORKDIR /plugins WORKDIR /config WORKDIR / +RUN apk add --no-cache fuse3 + COPY --from=builder /usr/local/cargo/bin/judger / COPY judger/nsjail-3.1 / diff --git a/judger/src/filesystem/adapter/fuse.rs b/judger/src/filesystem/adapter/fuse.rs index 90202c9..0e5b220 100644 --- a/judger/src/filesystem/adapter/fuse.rs +++ b/judger/src/filesystem/adapter/fuse.rs @@ -55,11 +55,6 @@ where mount_options.uid(uid).gid(gid).force_readdir_plus(true); - // FIXME: this panic in container - // - // additionally, libfuse report: `find fusermount3 binary failed` - metadata(path.as_ref()).await.expect("calling libc::mkdtemp actually creates the directory on host"); - Session::new(mount_options) .mount_with_unprivileged(self, path.as_ref()) .await diff --git a/judger/src/main.rs b/judger/src/main.rs index b9a82e6..13c7a51 100644 --- a/judger/src/main.rs +++ b/judger/src/main.rs @@ -10,6 +10,10 @@ pub use config::CONFIG; use grpc::judger::judger_server::JudgerServer; use server::Server; +#[cfg(not(debug_assertions))] +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + type Result = std::result::Result; #[tokio::main] diff --git a/judger/src/sandbox/monitor/mem_cpu.rs b/judger/src/sandbox/monitor/mem_cpu.rs index 057a528..2856d7d 100644 --- a/judger/src/sandbox/monitor/mem_cpu.rs +++ b/judger/src/sandbox/monitor/mem_cpu.rs @@ -111,8 +111,6 @@ impl super::Monitor for Monitor { /// This method is cancellation safe async fn wait_exhaust(&mut self) -> MonitorKind { let reason = self.monitor_task.as_mut().unwrap().await.unwrap(); - // optimistic kill(`SIGKILL`) the process inside - self.cgroup.kill().expect("cgroup.kill does not exist"); reason } fn poll_exhaust(&mut self) -> Option { diff --git a/judger/src/sandbox/process/nsjail.rs b/judger/src/sandbox/process/nsjail.rs index b57b2d7..5d07766 100644 --- a/judger/src/sandbox/process/nsjail.rs +++ b/judger/src/sandbox/process/nsjail.rs @@ -37,7 +37,8 @@ pub struct BaseArg; impl Argument for BaseArg { fn get_args(self) -> impl Iterator> { let mut args = vec![ - Cow::Borrowed(OsStr::from_bytes(b"-Me")), + // FIXME: MODE_STANDALONE_ONCE would might cause sandbox to continue running after process exit, check if that's true + Cow::Borrowed(OsStr::from_bytes(b"-Mo")), Cow::Borrowed(OsStr::from_bytes(b"-l")), #[cfg(not(debug_assertions))] Cow::Borrowed(OsStr::from_bytes(b"/dev/null")), @@ -73,8 +74,6 @@ impl<'a> Argument for CGroupMountArg<'a> { Cow::Borrowed(OsStr::from_bytes(b"0")), Cow::Borrowed(OsStr::from_bytes(b"--cgroup_cpu_parent")), Cow::Owned(OsString::from(self.cg_name)), - // Cow::Borrowed(OsStr::from_bytes(b"--cgroupv2_mount")), - // Cow::Owned(OsString::from(self.cg_name)), ], false => vec![ Cow::Borrowed(OsStr::from_bytes(b"--disable_clone_newcgroup")),