From 40c1004a58e42c2a3dc763d3fca57ad934818ac5 Mon Sep 17 00:00:00 2001 From: S1ro1 Date: Sun, 14 Dec 2025 23:12:59 +0100 Subject: [PATCH 1/7] Feat: fix sysinfo --- src/libkernelbot/report.py | 2 ++ src/libkernelbot/run_eval.py | 19 +++++++++++++++++++ src/libkernelbot/submission.py | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/libkernelbot/report.py b/src/libkernelbot/report.py index 2b48bfae..70f91487 100644 --- a/src/libkernelbot/report.py +++ b/src/libkernelbot/report.py @@ -295,9 +295,11 @@ def generate_system_info(system: SystemInfo): Running on: * GPU: `{system.gpu}` * CPU: `{system.cpu}` +* Device count: `{system.device_count}` * Runtime: `{system.runtime}` * Platform: `{system.platform}` * Torch: `{system.torch}` +* Hostname: `{system.hostname}` """ diff --git a/src/libkernelbot/run_eval.py b/src/libkernelbot/run_eval.py index 2cd6b397..2264cdac 100644 --- a/src/libkernelbot/run_eval.py +++ b/src/libkernelbot/run_eval.py @@ -5,6 +5,7 @@ import functools import json import os +import re import shlex import shutil import subprocess @@ -66,6 +67,7 @@ class SystemInfo: runtime: str = '' # Whether CUDA or ROCm platform: str = '' # Platform string of the machine torch: str = '' # Torch version + hostname: str = '' # Hostname of the machine # fmt: on @@ -573,6 +575,23 @@ def run_single_evaluation( return run_program(call, seed=seed, timeout=timeout, multi_gpu=multi_gpu), None +def container_id_from_cgroup() -> str | None: + txt = Path("/proc/self/cgroup").read_text(errors="ignore") + # docker: .../docker/<64hex> + m = re.search(r"/docker/([0-9a-f]{64})", txt) + if m: + return m.group(1) + # containerd: .../kubepods.../<64hex> + m = re.search(r"([0-9a-f]{64})", txt) + if m: + return m.group(1) + # sometimes it's 32hex + m = re.search(r"([0-9a-f]{32})", txt) + if m: + return m.group(1) + return None + + def make_system_info() -> SystemInfo: # noqa: C901 info = SystemInfo() try: diff --git a/src/libkernelbot/submission.py b/src/libkernelbot/submission.py index d659878d..0627785b 100644 --- a/src/libkernelbot/submission.py +++ b/src/libkernelbot/submission.py @@ -180,7 +180,7 @@ def compute_score(result: FullResult, task: LeaderboardTask, submission_id: int) num_benchmarks, ) raise KernelBotError( - f"Expected submission to have exactly one benchmark," f"got {num_benchmarks}." + f"Expected submission to have exactly one benchmark,got {num_benchmarks}." ) score = float(result.runs["leaderboard"].run.result["benchmark.0.mean"]) / 1e9 else: From 5168de0d158a89d6cb7c6d3b3babb3cff0c81af2 Mon Sep 17 00:00:00 2001 From: S1ro1 Date: Sun, 14 Dec 2025 23:13:49 +0100 Subject: [PATCH 2/7] Feat: fix sysinfo --- src/libkernelbot/run_eval.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/libkernelbot/run_eval.py b/src/libkernelbot/run_eval.py index 2264cdac..aec59f95 100644 --- a/src/libkernelbot/run_eval.py +++ b/src/libkernelbot/run_eval.py @@ -5,9 +5,9 @@ import functools import json import os -import re import shlex import shutil +import socket import subprocess import tempfile import time @@ -575,23 +575,6 @@ def run_single_evaluation( return run_program(call, seed=seed, timeout=timeout, multi_gpu=multi_gpu), None -def container_id_from_cgroup() -> str | None: - txt = Path("/proc/self/cgroup").read_text(errors="ignore") - # docker: .../docker/<64hex> - m = re.search(r"/docker/([0-9a-f]{64})", txt) - if m: - return m.group(1) - # containerd: .../kubepods.../<64hex> - m = re.search(r"([0-9a-f]{64})", txt) - if m: - return m.group(1) - # sometimes it's 32hex - m = re.search(r"([0-9a-f]{32})", txt) - if m: - return m.group(1) - return None - - def make_system_info() -> SystemInfo: # noqa: C901 info = SystemInfo() try: @@ -650,6 +633,8 @@ def make_system_info() -> SystemInfo: # noqa: C901 pass import platform + info.hostname = socket.gethostname() + info.platform = platform.platform() return info From 4c882d49e90ad43556b30b0ce13fc31df576bd0d Mon Sep 17 00:00:00 2001 From: Matej Sirovatka <54212263+S1ro1@users.noreply.github.com> Date: Sun, 14 Dec 2025 23:21:15 +0100 Subject: [PATCH 3/7] Update src/libkernelbot/submission.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/libkernelbot/submission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libkernelbot/submission.py b/src/libkernelbot/submission.py index 0a3ffd67..805f7435 100644 --- a/src/libkernelbot/submission.py +++ b/src/libkernelbot/submission.py @@ -180,7 +180,7 @@ def compute_score(result: FullResult, task: LeaderboardTask, submission_id: int) num_benchmarks, ) raise KernelBotError( - f"Expected submission to have exactly one benchmark,got {num_benchmarks}." + f"Expected submission to have exactly one benchmark, got {num_benchmarks}." ) score = float(result.runs["leaderboard"].run.result["benchmark.0.mean"]) / 1e9 else: From 6a95692c30e547cccb27df19928502ca61bb818f Mon Sep 17 00:00:00 2001 From: S1ro1 Date: Sun, 14 Dec 2025 23:46:09 +0100 Subject: [PATCH 4/7] Feat: tests --- tests/test_report.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/test_report.py b/tests/test_report.py index 9006a98e..ae3afd25 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -35,6 +35,8 @@ def sample_system_info() -> SystemInfo: runtime="CUDA", platform="Linux-5.15.0", torch="2.0.1+cu118", + hostname="test-hostname", + device_count=1, ) @@ -504,9 +506,11 @@ def test_generate_report_test_failure(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Text( text="# Testing failed\n" @@ -536,9 +540,11 @@ def test_generate_report_benchmark_failure(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -571,9 +577,11 @@ def test_generate_report_benchmark_failure(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -607,9 +615,11 @@ def test_generate_report_leaderboard_failure(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -633,9 +643,11 @@ def test_generate_report_leaderboard_failure(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -646,10 +658,7 @@ def test_generate_report_leaderboard_failure(sample_full_result: FullResult): "> Division by zero", ), Text( - text="# Running failed\n" - "Command ```bash\n" - "./test```\n" - "**timed out** after 10.00 seconds." + text="# Running failed\nCommand ```bash\n./test```\n**timed out** after 10.00 seconds." ), Log(header="Program stdout", content="log stdout"), ] @@ -676,9 +685,11 @@ def test_generate_report_profile(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -688,11 +699,9 @@ def test_generate_report_profile(sample_full_result: FullResult): "❌ Test division\n" "> Division by zero", ), - Log(header='Profiling Benchmark', content=' Profile report\n'), + Log(header="Profiling Benchmark", content=" Profile report\n"), Link("NSight profiling output", "Download from GitHub", "https://example.com"), - File(name='profile-Benchmark.zip', - message='NSight report - Benchmark', - content=b''), + File(name="profile-Benchmark.zip", message="NSight report - Benchmark", content=b""), ] From 112ab1529e7116d09248f48daabe18435646bbfe Mon Sep 17 00:00:00 2001 From: S1ro1 Date: Sun, 14 Dec 2025 23:49:26 +0100 Subject: [PATCH 5/7] Feat: tests --- tests/test_backend.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_backend.py b/tests/test_backend.py index 94cc2795..3b146610 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -77,9 +77,11 @@ async def test_handle_submission(bot: backend.KernelBackend, task_directory): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", From ea8d070a1654c4971019872007e21ff0a3613b47 Mon Sep 17 00:00:00 2001 From: S1ro1 Date: Mon, 15 Dec 2025 00:00:54 +0100 Subject: [PATCH 6/7] Feat: tests --- tests/test_backend.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_backend.py b/tests/test_backend.py index 3b146610..7a7df4ee 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -215,6 +215,7 @@ async def test_submit_leaderboard(bot: backend.KernelBackend, task_directory): "platform": "Linux-5.15.0", "runtime": "CUDA", "torch": "2.0.1+cu118", + "hostname": "test-hostname", }, } ], @@ -364,6 +365,7 @@ async def test_submit_full(bot: backend.KernelBackend, task_directory): "platform": "Linux-5.15.0", "runtime": "CUDA", "torch": "2.0.1+cu118", + "hostname": "test-hostname", }, }, ], From ffb8a8bf7bcc260406f22f9ae6a3af301de9b5d0 Mon Sep 17 00:00:00 2001 From: S1ro1 Date: Mon, 15 Dec 2025 00:05:48 +0100 Subject: [PATCH 7/7] Feat: tests --- tests/test_backend.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_backend.py b/tests/test_backend.py index 7a7df4ee..f69170c5 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -322,6 +322,7 @@ async def test_submit_full(bot: backend.KernelBackend, task_directory): "platform": "Linux-5.15.0", "runtime": "CUDA", "torch": "2.0.1+cu118", + "hostname": "test-hostname", }, }, {