diff --git a/README.md b/README.md index 2486b0a7..69b162da 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,8 @@ The autotester currently supports testers for the following languages and testin - [RackUnit](https://docs.racket-lang.org/rackunit/) - `R` - [TestThat](https://testthat.r-lib.org/) +- `rust' + - [Cargo Test](https://doc.rust-lang.org/book/ch11-01-writing-tests.html) - `custom` - see more information [here](#the-custom-tester) @@ -160,6 +162,9 @@ Installing each tester will also install the following additional packages (syst - racket - `R` - R +- `Rust` + - Rust and Cargo (rustup) + - Cargo Nextest - `custom` - none diff --git a/server/autotest_server/testers/__init__.py b/server/autotest_server/testers/__init__.py index 1ca645c2..c0a8d91e 100644 --- a/server/autotest_server/testers/__init__.py +++ b/server/autotest_server/testers/__init__.py @@ -1,6 +1,6 @@ import os -_TESTERS = ("custom", "haskell", "java", "jupyter", "py", "pyta", "r", "racket") +_TESTERS = ("custom", "haskell", "java", "jupyter", "py", "pyta", "r", "racket", "rust") def install(testers=_TESTERS): diff --git a/server/autotest_server/testers/rust/__init__.py b/server/autotest_server/testers/rust/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/autotest_server/testers/rust/requirements.system b/server/autotest_server/testers/rust/requirements.system new file mode 100755 index 00000000..14e9b2e3 --- /dev/null +++ b/server/autotest_server/testers/rust/requirements.system @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# The main repository for APT seems to be fairly old (1.65). +# Building cargo-nextest from source requires Rust 1.70, so I'm using rustup instead of a package manager. + +# We need a few things for our rust target. +# - curl for downloading rustup. Alternatively we can use wget. +# - rustup for installing rust and installing cargo-nextest. +# - cargo-nextest for machine readable test output. + +# Install Curl +if ! dpkg -l curl &> /dev/null; then + apt-get -y update + + DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' curl +fi + +# Install Rust +if ! command -v cargo &> /dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + + # Add cargo to the path. + . "$HOME/.cargo/env" +fi + +# Install Nextest +# For justification as to why we're using nextest, take a look at rust_tester.py. +if ! command -v cargo-nextest &> /dev/null; then + cargo install cargo-nextest --locked +fi diff --git a/server/autotest_server/testers/rust/rust_tester.py b/server/autotest_server/testers/rust/rust_tester.py new file mode 100644 index 00000000..d0ddf7c0 --- /dev/null +++ b/server/autotest_server/testers/rust/rust_tester.py @@ -0,0 +1,160 @@ +import os +import subprocess +from typing import Type, Optional +import json +from ..tester import Tester, Test, TestError +from ..specs import TestSpecs +from pathlib import Path +import re + +TEST_IDENTIFIER_REGEX = re.compile("^((?P.*)\\$)?(?P.*)$") + + +def parse_test_identifier(identifier: str) -> tuple[str, str]: + result = TEST_IDENTIFIER_REGEX.match(identifier) + + return result.group("exec"), result.group("test") + + +class RustTest(Test): + def __init__(self, tester: "RustTester", executable: str, test: str, success: bool, message: str) -> None: + self.executable = executable + self.test = test + self.success = success + self.message = message + + super().__init__(tester) + + @property + def test_name(self) -> str: + return self.test + + def run(self) -> str: + if self.success: + return self.passed(message=self.message) + else: + return self.failed(message=self.message) + + +def parse_adjacent_json(text: str) -> list[dict]: + skip = {"\n", "\r"} + + i = 0 + decoder = json.JSONDecoder() + + items = [] + + while i < len(text): + value, i = decoder.raw_decode(text, i) + + items.append(value) + + while i < len(text) and text[i] in skip: + i += 1 + + return items + + +class RustTester(Tester): + def __init__(self, specs: TestSpecs, test_class: Type[RustTest] = RustTest) -> None: + super().__init__(specs, test_class) + + def parse_test_events(self, items: list[dict]) -> list[RustTest]: + tests = [] + + for item in items: + # Ignore suite events. + if item["type"] != "test": + continue + + event = item["event"] + + finished_events = {"failed", "ok"} + + if event not in finished_events: + continue + + executable, test = parse_test_identifier(item["name"]) + + output = item["stdout"] if event == "failed" else "" + + tests.append(self.test_class(self, executable, test, event == "ok", output)) + + return tests + + # This should likely be moved to setup.py's create_environment. + def rust_env(self) -> dict: + # Hint to /bin/sh that it should look in $HOME/.cargo/bin. + # Despite .profile pointing to this directory, /bin/dash does not want to acknowledge it. + # There is conflicting information online as to whether-or-not .profile is respected by dash. + rust_home_path = os.path.join(Path.home(), ".cargo", "bin") + + env = os.environ.copy() + + env["PATH"] = rust_home_path + ":" + env["PATH"] + + return env + + def compile_rust_tests(self, directory: str) -> subprocess.CompletedProcess: + command = ["cargo", "nextest", "run", "--no-run", "--color", "never"] + + env = self.rust_env() + + return subprocess.run(command, cwd=directory, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # We need a way to get machine-readable output from cargo test (why are we using nextest?). + # Here are the options I've explored: + # - cargo-test by default outputs human-readable strings. + # These strings might change in format over time, and I figure one tests stdout could bleed into another. + # - cargo-test supports machine-readable strings (via `cargo test -- --format json`). + # This option is nightly rust only, and enforcing nightly rust could cause other issues. + # - cargo-nextest supports experimental machine-readable output. + # While machine-readable output is experimental, it's based on yhr libtest standard that cargo test would use. + # nextest should also interact very similarly to `cargo test`. It should be very simple to swap to cargo-test. + # It's also reliable and only requires Rust 1.36 or earlier for running. + def run_rust_tests(self, directory: str, module: Optional[str]) -> subprocess.CompletedProcess: + command = ["cargo", "nextest", "run", "--no-fail-fast", "--message-format", "libtest-json", "--color", "never"] + + # Prevent CLI options from being propagated. + if module is not None and "-" not in module: + command.append(module) + + # Machine-readable output is experimental with Nextest. + env = self.rust_env() + env["NEXTEST_EXPERIMENTAL_LIBTEST_JSON"] = "1" + + return subprocess.run(command, cwd=directory, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def run_and_parse_rust_tests(self, directory: str, module: Optional[str]) -> list[RustTest]: + test_results = self.run_rust_tests(directory, module) + + json_string = test_results.stdout.decode("utf-8") + test_events = parse_adjacent_json(json_string) + + return self.parse_test_events(test_events) + + @Tester.run_decorator + def run(self) -> None: + # Awkwardly, cargo doesn't have a great way of running files. + # Instead, it can run all the tests in a module (which is named after a file barring .rs and the path). + + try: + compile_result = self.compile_rust_tests(".") + + if compile_result.returncode != 0: + raise TestError(compile_result.stderr.decode("utf-8")) + except subprocess.CalledProcessError as e: + raise TestError(e) + + module = self.specs.get("test_data", "test_module") + + if module is not None: + module = module.strip() + + if module == "": + module = None + + tests = self.run_and_parse_rust_tests(".", module) + + for test in tests: + print(test.run(), flush=True) diff --git a/server/autotest_server/testers/rust/settings_schema.json b/server/autotest_server/testers/rust/settings_schema.json new file mode 100644 index 00000000..d56d3830 --- /dev/null +++ b/server/autotest_server/testers/rust/settings_schema.json @@ -0,0 +1,52 @@ +{ + "type": "object", + "properties": { + "tester_type": { + "type": "string", + "enum": [ + "rust" + ] + }, + "test_data": { + "title": "Test Groups", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "timeout" + ], + "properties": { + "test_module": { + "title": "Test module name", + "type": "string", + "minLength": 0 + }, + "category": { + "title": "Category", + "type": "array", + "items": { + "$ref": "#/definitions/test_data_categories" + }, + "uniqueItems": true + }, + "timeout": { + "title": "Timeout", + "type": "integer", + "default": 30 + }, + "feedback_file_names": { + "title": "Feedback files", + "type": "array", + "items": { + "type": "string" + } + }, + "extra_info": { + "$ref": "#/definitions/extra_group_data" + } + } + } + } + } +} diff --git a/server/autotest_server/testers/rust/setup.py b/server/autotest_server/testers/rust/setup.py new file mode 100644 index 00000000..59081122 --- /dev/null +++ b/server/autotest_server/testers/rust/setup.py @@ -0,0 +1,16 @@ +import os +import json +import subprocess + + +def create_environment(settings_, env_dir, default_env_dir): + return {"PYTHON": os.path.join(default_env_dir, "bin", "python3")} + + +def settings(): + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: + return json.load(f) + + +def install(): + subprocess.run(os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system"), check=True)