diff --git a/src/libkernelbot/report.py b/src/libkernelbot/report.py index 2b48bfae..70f91487 100644 --- a/src/libkernelbot/report.py +++ b/src/libkernelbot/report.py @@ -295,9 +295,11 @@ def generate_system_info(system: SystemInfo): Running on: * GPU: `{system.gpu}` * CPU: `{system.cpu}` +* Device count: `{system.device_count}` * Runtime: `{system.runtime}` * Platform: `{system.platform}` * Torch: `{system.torch}` +* Hostname: `{system.hostname}` """ diff --git a/src/libkernelbot/run_eval.py b/src/libkernelbot/run_eval.py index 2cd6b397..aec59f95 100644 --- a/src/libkernelbot/run_eval.py +++ b/src/libkernelbot/run_eval.py @@ -7,6 +7,7 @@ import os import shlex import shutil +import socket import subprocess import tempfile import time @@ -66,6 +67,7 @@ class SystemInfo: runtime: str = '' # Whether CUDA or ROCm platform: str = '' # Platform string of the machine torch: str = '' # Torch version + hostname: str = '' # Hostname of the machine # fmt: on @@ -631,6 +633,8 @@ def make_system_info() -> SystemInfo: # noqa: C901 pass import platform + info.hostname = socket.gethostname() + info.platform = platform.platform() return info diff --git a/src/libkernelbot/submission.py b/src/libkernelbot/submission.py index 3b5fa703..805f7435 100644 --- a/src/libkernelbot/submission.py +++ b/src/libkernelbot/submission.py @@ -180,7 +180,7 @@ def compute_score(result: FullResult, task: LeaderboardTask, submission_id: int) num_benchmarks, ) raise KernelBotError( - f"Expected submission to have exactly one benchmark," f"got {num_benchmarks}." + f"Expected submission to have exactly one benchmark, got {num_benchmarks}." ) score = float(result.runs["leaderboard"].run.result["benchmark.0.mean"]) / 1e9 else: diff --git a/tests/test_backend.py b/tests/test_backend.py index 94cc2795..f69170c5 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -77,9 +77,11 @@ async def test_handle_submission(bot: backend.KernelBackend, task_directory): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -213,6 +215,7 @@ async def test_submit_leaderboard(bot: backend.KernelBackend, task_directory): "platform": "Linux-5.15.0", "runtime": "CUDA", "torch": "2.0.1+cu118", + "hostname": "test-hostname", }, } ], @@ -319,6 +322,7 @@ async def test_submit_full(bot: backend.KernelBackend, task_directory): "platform": "Linux-5.15.0", "runtime": "CUDA", "torch": "2.0.1+cu118", + "hostname": "test-hostname", }, }, { @@ -362,6 +366,7 @@ async def test_submit_full(bot: backend.KernelBackend, task_directory): "platform": "Linux-5.15.0", "runtime": "CUDA", "torch": "2.0.1+cu118", + "hostname": "test-hostname", }, }, ], diff --git a/tests/test_report.py b/tests/test_report.py index 9006a98e..ae3afd25 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -35,6 +35,8 @@ def sample_system_info() -> SystemInfo: runtime="CUDA", platform="Linux-5.15.0", torch="2.0.1+cu118", + hostname="test-hostname", + device_count=1, ) @@ -504,9 +506,11 @@ def test_generate_report_test_failure(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Text( text="# Testing failed\n" @@ -536,9 +540,11 @@ def test_generate_report_benchmark_failure(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -571,9 +577,11 @@ def test_generate_report_benchmark_failure(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -607,9 +615,11 @@ def test_generate_report_leaderboard_failure(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -633,9 +643,11 @@ def test_generate_report_leaderboard_failure(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -646,10 +658,7 @@ def test_generate_report_leaderboard_failure(sample_full_result: FullResult): "> Division by zero", ), Text( - text="# Running failed\n" - "Command ```bash\n" - "./test```\n" - "**timed out** after 10.00 seconds." + text="# Running failed\nCommand ```bash\n./test```\n**timed out** after 10.00 seconds." ), Log(header="Program stdout", content="log stdout"), ] @@ -676,9 +685,11 @@ def test_generate_report_profile(sample_full_result: FullResult): "Running on:\n" "* GPU: `NVIDIA RTX 4090`\n" "* CPU: `Intel i9-12900K`\n" + "* Device count: `1`\n" "* Runtime: `CUDA`\n" "* Platform: `Linux-5.15.0`\n" "* Torch: `2.0.1+cu118`\n" + "* Hostname: `test-hostname`\n" ), Log( header="✅ Passed 3/3 tests", @@ -688,11 +699,9 @@ def test_generate_report_profile(sample_full_result: FullResult): "❌ Test division\n" "> Division by zero", ), - Log(header='Profiling Benchmark', content=' Profile report\n'), + Log(header="Profiling Benchmark", content=" Profile report\n"), Link("NSight profiling output", "Download from GitHub", "https://example.com"), - File(name='profile-Benchmark.zip', - message='NSight report - Benchmark', - content=b''), + File(name="profile-Benchmark.zip", message="NSight report - Benchmark", content=b""), ]