From e651b08a846043e9c71bf51cac9313cf2d49d171 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 8 Sep 2025 15:02:40 +0100 Subject: [PATCH 01/11] fix(rust): unallow exit --- entrypoint.sh | 2 +- isolate.sh | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100755 isolate.sh diff --git a/entrypoint.sh b/entrypoint.sh index 41201855..95476cba 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -33,5 +33,5 @@ fi if test "$CODE_EDITOR_RUN_ONLY"; then cargo run --manifest-path "solutions/$EXERCISE/Cargo.toml" -- "$@" else - cargo test --manifest-path "tests/${EXERCISE}_test/Cargo.toml" + cargo --config 'target."cfg(all())".runner="./isolate.sh"' test --manifest-path "tests/${EXERCISE}_test/Cargo.toml" fi diff --git a/isolate.sh b/isolate.sh new file mode 100755 index 00000000..d2eb896f --- /dev/null +++ b/isolate.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Use: +# cargo --config 'target."cfg(all())".runner="./isolate.sh"' test [args] + +set -u + +bin="${1:-}"; shift || true +[[ -n "${bin:-}" ]] || { echo "[wrapper] missing test binary" >&2; exit 2; } + +# Strip Cargo-only flags that confuse the test binary (e.g., --message-format) +filter_args() { + local -a out=(); local skip=0 + for a in "$@"; do + if [[ $skip -eq 1 ]]; then skip=0; continue; fi + case "$a" in + --message-format|--format) skip=1 ;; # drop the next token + --message-format=*|--format=*) ;; # drop inline forms + *) out+=("$a") ;; + esac + done + printf '%s\n' "${out[@]}" +} +filtered=($(filter_args "$@")) + +tmpdir="$(mktemp -d -t strictwrap.XXXXXX)"; trap 'rm -rf "$tmpdir"' EXIT +expected="$tmpdir/expected.txt" +actual="$tmpdir/actual.txt" +logfile="$tmpdir/run.out" + +# 1) Ask the harness which tests it *will* run (respects filters in "$@") +if ! "$bin" --list "${filtered[@]}" >"$tmpdir/list.txt" 2>/dev/null; then + echo "[wrapper] unable to list tests; cannot verify harness" >&2 + exit 1 +fi +# Lines look like: `path::to::name: test` (optionally with " (ignored)") +awk -F': test' '/: test/{print $1}' "$tmpdir/list.txt" | sed 's/[[:space:]]*$//' > "$expected" + +# 2) Run the suite normally and capture output + real exit code +( + "$bin" "${filtered[@]}" 2>&1 + echo "__RC__$?" +) | tee "$logfile" >/dev/null +rc="$(awk -F'__RC__' '/__RC__/ {v=$2} END{print v+0}' "$logfile")" + +# 3) Collect tests that actually produced a result line: +# Matches lines like: `test foo::bar ... ok|ignored|FAILED` +awk ' + /^test[[:space:]][^ ]/ { + line=$0 + sub(/^test[[:space:]]+/, "", line) + sub(/[[:space:]]+\.\.\..*$/, "", line) + print line + } +' "$logfile" | sed 's/[[:space:]]*$//' > "$actual" + +# 4) Decide: require (a) rc==0, (b) final summary ok, (c) every expected test appeared +if [[ "$rc" -eq 0 ]] && grep -Eq '^test result: ok\.' "$logfile"; then + sort -u "$expected" -o "$expected" + sort -u "$actual" -o "$actual" + # If no tests were expected (filters matched none), that's fine too. + if comm -23 "$expected" "$actual" | read -r _; then + # there were missing tests → failure + exit 1 + else + exit 0 + fi +fi + +exit 1 From 6c044863baafd7116a474b327e96ea51a04eae90 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 8 Sep 2025 15:12:05 +0100 Subject: [PATCH 02/11] use mapfile --- isolate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isolate.sh b/isolate.sh index d2eb896f..baaf689a 100755 --- a/isolate.sh +++ b/isolate.sh @@ -20,7 +20,7 @@ filter_args() { done printf '%s\n' "${out[@]}" } -filtered=($(filter_args "$@")) +mapfile -t filtered < <(filter_args "$@") tmpdir="$(mktemp -d -t strictwrap.XXXXXX)"; trap 'rm -rf "$tmpdir"' EXIT expected="$tmpdir/expected.txt" From eedb551f9478d76abf6b41f731f9717a6504e5ee Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 8 Sep 2025 15:20:12 +0100 Subject: [PATCH 03/11] add isolate.sh to Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e2fa2a83..6755e18b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,5 +23,6 @@ LABEL org.opencontainers.image.description="01 Edu - Rust Test Image" LABEL org.opencontainers.image.licenses=MIT COPY entrypoint.sh ./ +COPY isolate.sh ./ ENTRYPOINT ["/app/entrypoint.sh"] From 7ed5acfe5665e9c07645bcaaecf5d33261b430ed Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 8 Sep 2025 15:39:48 +0100 Subject: [PATCH 04/11] change isolate.sh folder on Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6755e18b..21ab70b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,6 @@ LABEL org.opencontainers.image.description="01 Edu - Rust Test Image" LABEL org.opencontainers.image.licenses=MIT COPY entrypoint.sh ./ -COPY isolate.sh ./ +COPY isolate.sh ./jail/student/ ENTRYPOINT ["/app/entrypoint.sh"] From b60e413615a7915616996bfb4d1db521f8c117e9 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 8 Sep 2025 15:48:05 +0100 Subject: [PATCH 05/11] change isolate.sh folder on Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 21ab70b8..de8bafc6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,6 @@ LABEL org.opencontainers.image.description="01 Edu - Rust Test Image" LABEL org.opencontainers.image.licenses=MIT COPY entrypoint.sh ./ -COPY isolate.sh ./jail/student/ +COPY isolate.sh ./jail/ ENTRYPOINT ["/app/entrypoint.sh"] From a5b31a6e55201968e9493ca25b359317b422990e Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 8 Sep 2025 15:54:00 +0100 Subject: [PATCH 06/11] change runner path --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 95476cba..be4bbd4d 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -33,5 +33,5 @@ fi if test "$CODE_EDITOR_RUN_ONLY"; then cargo run --manifest-path "solutions/$EXERCISE/Cargo.toml" -- "$@" else - cargo --config 'target."cfg(all())".runner="./isolate.sh"' test --manifest-path "tests/${EXERCISE}_test/Cargo.toml" + cargo --config 'target."cfg(all())".runner="/jail/isolate.sh"' test --manifest-path "tests/${EXERCISE}_test/Cargo.toml" fi From e8a8aeb50d3f176a5c94142f51b39f7b0245a627 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 8 Sep 2025 16:00:02 +0100 Subject: [PATCH 07/11] not a relative path.. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index de8bafc6..73e075fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,6 @@ LABEL org.opencontainers.image.description="01 Edu - Rust Test Image" LABEL org.opencontainers.image.licenses=MIT COPY entrypoint.sh ./ -COPY isolate.sh ./jail/ +COPY isolate.sh /jail/ ENTRYPOINT ["/app/entrypoint.sh"] From 3b0f382d4e4429936a002e83214720ac0d408c17 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 8 Sep 2025 18:00:08 +0100 Subject: [PATCH 08/11] changed paths again. i love docker --- Dockerfile | 2 +- entrypoint.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 73e075fc..6755e18b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,6 @@ LABEL org.opencontainers.image.description="01 Edu - Rust Test Image" LABEL org.opencontainers.image.licenses=MIT COPY entrypoint.sh ./ -COPY isolate.sh /jail/ +COPY isolate.sh ./ ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/entrypoint.sh b/entrypoint.sh index be4bbd4d..d79bbc08 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -10,6 +10,7 @@ CODE_EDITOR_RUN_ONLY="${CODE_EDITOR_RUN_ONLY:-$EXAM_RUN_ONLY}" CODE_EDITOR_MODE="${CODE_EDITOR_MODE:-$EXAM_MODE}" cp -a /app/tests . +cp -a /app/isolate.sh . cp -a student solutions if test "$CODE_EDITOR_MODE"; then @@ -33,5 +34,5 @@ fi if test "$CODE_EDITOR_RUN_ONLY"; then cargo run --manifest-path "solutions/$EXERCISE/Cargo.toml" -- "$@" else - cargo --config 'target."cfg(all())".runner="/jail/isolate.sh"' test --manifest-path "tests/${EXERCISE}_test/Cargo.toml" + cargo --config 'target."cfg(all())".runner="./isolate.sh"' test --manifest-path "tests/${EXERCISE}_test/Cargo.toml" fi From a2abf2fd3734c9314e3882bbd7f90d9c7b442227 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 8 Sep 2025 18:15:23 +0100 Subject: [PATCH 09/11] should work now - didn't take should_panic tests in consideration --- isolate.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/isolate.sh b/isolate.sh index baaf689a..472a09e2 100755 --- a/isolate.sh +++ b/isolate.sh @@ -32,7 +32,7 @@ if ! "$bin" --list "${filtered[@]}" >"$tmpdir/list.txt" 2>/dev/null; then echo "[wrapper] unable to list tests; cannot verify harness" >&2 exit 1 fi -# Lines look like: `path::to::name: test` (optionally with " (ignored)") +# Lines look like: `path::to::name: test` (sometimes with " (ignored)") awk -F': test' '/: test/{print $1}' "$tmpdir/list.txt" | sed 's/[[:space:]]*$//' > "$expected" # 2) Run the suite normally and capture output + real exit code @@ -44,11 +44,14 @@ rc="$(awk -F'__RC__' '/__RC__/ {v=$2} END{print v+0}' "$logfile")" # 3) Collect tests that actually produced a result line: # Matches lines like: `test foo::bar ... ok|ignored|FAILED` +# Normalize names by stripping the " - should panic[ with `...`]" suffix. awk ' /^test[[:space:]][^ ]/ { line=$0 - sub(/^test[[:space:]]+/, "", line) - sub(/[[:space:]]+\.\.\..*$/, "", line) + sub(/^test[[:space:]]+/, "", line) # drop leading "test " + sub(/[[:space:]]+\.\.\..*$/, "", line) # drop " ... ok/FAILED/ignored" + # Remove should_panic annotation added to pretty output: + sub(/[[:space:]]+- should panic( with `[^`]*`)?$/, "", line) print line } ' "$logfile" | sed 's/[[:space:]]*$//' > "$actual" From 1b4d39b0089fd6f6abd29c14c49fa985168b8366 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 29 Sep 2025 15:17:12 +0100 Subject: [PATCH 10/11] fix(rust-tests): add logging if some tests aren't ran --- isolate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/isolate.sh b/isolate.sh index 472a09e2..06e9cb4a 100755 --- a/isolate.sh +++ b/isolate.sh @@ -63,6 +63,7 @@ if [[ "$rc" -eq 0 ]] && grep -Eq '^test result: ok\.' "$logfile"; then # If no tests were expected (filters matched none), that's fine too. if comm -23 "$expected" "$actual" | read -r _; then # there were missing tests → failure + echo "Some tests weren't ran for the exercise `$EXERCISE`. Perhaps the solution forcefully exits?" exit 1 else exit 0 From 4720a55e0afbbdbf2510e39898b43c5c2ee39529 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 29 Sep 2025 15:22:14 +0100 Subject: [PATCH 11/11] fix(rust-tests): escape backticks on isolate.sh --- isolate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isolate.sh b/isolate.sh index 06e9cb4a..6f0414e5 100755 --- a/isolate.sh +++ b/isolate.sh @@ -63,7 +63,7 @@ if [[ "$rc" -eq 0 ]] && grep -Eq '^test result: ok\.' "$logfile"; then # If no tests were expected (filters matched none), that's fine too. if comm -23 "$expected" "$actual" | read -r _; then # there were missing tests → failure - echo "Some tests weren't ran for the exercise `$EXERCISE`. Perhaps the solution forcefully exits?" + echo "Some tests weren't ran for the exercise \`$EXERCISE\`. Perhaps the solution forcefully exits?" exit 1 else exit 0