Add automated QEMU-based testing to CI (#1041)

* Added `qemu_test` application that automatically runs all tests
  and returns a specific exit code from QEMU based on test output.

* Added a `test` Make target that enables and runs `qemu_test`.
  * Added a CI workflow that runs the `test` target

* Changed `test_mlx5`, `test_ixgbe`, and `test_block_io`
  to return an exit code of 0 if the required devices
  aren't connected.

* Changed `test_identity_mapping`, `test_aligned_page_allocation`,
  and `test_filerw` to fail rather than print if they encounter an error.

* Renamed `tls_test` to `test_tls` so it is detected by `qemu_test`.

* Changed `test_channel` and `test_tls` to run all tests
  rather than specifying specific tests in the arguments.

* Renamed `test_serial_echo` to `serial_echo` because it
   isn't really a test with defined success and failure conditions.

* Changed `test_task_cancel` to always return 0, because
  task cancellation is not yet implemented in the mainline.

* Skip `test_channel`, as it currently does not work.

Signed-off-by: Klimenty Tsoutsman <klim@tsoutsman.com>

.github/workflows/check-clippy.yaml

@@ -11,10 +11,10 @@ jobs:
         run: |
           git submodule update --init --recursive
       - name: "Install nasm"
-        run: sudo apt install nasm
+        run: |
+          sudo apt update
+          sudo apt install nasm
       - name: "Run Clippy"
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         working-directory: .
         run: |
           make clippy ARCH=x86_64

.github/workflows/docs.yaml

@@ -13,7 +13,9 @@ jobs:
         submodules: recursive
 
     - name: "Install nasm"
-      run: sudo apt install nasm
+      run: |
+        sudo apt update
+        sudo apt install nasm
 
     - name: Cache build artifacts
       uses: actions/cache@v3

.github/workflows/test.yaml

@@ -0,0 +1,20 @@
+name: QEMU Test
+on:
+  pull_request:
+    types: [synchronize, opened, reopened]
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: "Initialize git submodules"
+        run: |
+          git submodule update --init --recursive
+      - name: "Install dependencies"
+        run: |
+          sudo apt update
+          sudo apt install make gcc nasm pkg-config grub-pc-bin mtools xorriso qemu qemu-kvm wget
+      - name: "Run tests"
+        working-directory: .
+        run: make test
+        timeout-minutes: 10

Makefile

@@ -1079,3 +1079,14 @@ endif
 	@sudo cp -vf $(iso) /var/lib/tftpboot/theseus/
 	@sudo systemctl restart isc-dhcp-server 
 	@sudo systemctl restart tftpd-hpa
+
+test: export override QEMU_FLAGS += -device isa-debug-exit,iobase=0xf4,iosize=0x04
+test: export override QEMU_FLAGS += -nographic
+test: export override FEATURES =--features theseus_tests --features first_application/qemu_test
+test: $(iso)
+	# We exit with an exit code of 0 if QEMU's exit code is 17, and 2 otherwise.
+	# This is because `qemu_test` uses a value of 0x11 to indicate success.
+	$(QEMU_BIN) $(QEMU_FLAGS); \
+	EXIT_CODE=$$?; \
+	test $$EXIT_CODE -eq 17 && exit 0; \
+	exit 2

applications/qemu_test/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "qemu_test"
+version = "0.1.0"
+authors = ["Klim Tsoutsman <klim@tsoutsman.com>"]
+description = "Automated test runner"
+edition = "2021"
+
+[dependencies]
+app_io = { path = "../../kernel/app_io" }
+path = { path = "../../kernel/path" }
+qemu-exit = "3.0.2"
+spawn = { path = "../../kernel/spawn" }
+task = { path = "../../kernel/task" }

applications/qemu_test/src/lib.rs

@@ -0,0 +1,120 @@
+//! An automated test runner.
+//!
+//! The application assumes it is running in a QEMU virtual machine and exits
+//! from QEMU with different exit codes depending on whether the tests passed or
+//! failed.
+
+#![no_std]
+
+use alloc::{boxed::Box, string::String, vec::Vec};
+
+use app_io::{print, println};
+use path::Path;
+use qemu_exit::{QEMUExit, X86};
+use task::{ExitValue, KillReason};
+
+extern crate alloc;
+
+static QEMU_EXIT_HANDLE: X86 = X86::new(0xf4, 0x11);
+
+pub fn main(_: Vec<String>) -> isize {
+    task::set_kill_handler(Box::new(|_| {
+        QEMU_EXIT_HANDLE.exit_failure();
+    }))
+    .unwrap();
+
+    let dir = task::get_my_current_task()
+        .map(|t| t.get_namespace().dir().clone())
+        .expect("couldn't get namespace dir");
+
+    let object_files = dir.lock().list();
+
+    let test_paths = object_files
+        .into_iter()
+        .filter_map(|file_name| {
+            if file_name.starts_with("test_") {
+                // We must release the lock prior to calling `get_absolute_path` to avoid
+                // deadlock.
+                let file = dir.lock().get_file(file_name.as_ref()).unwrap();
+                let path = file.lock().get_absolute_path();
+                Some((file_name, Path::new(path)))
+            } else {
+                None
+            }
+        })
+        .collect::<Vec<_>>();
+
+    let total = test_paths.len();
+    println!("running {} tests", total);
+
+    let mut num_ignored = 0;
+    let mut num_failed = 0;
+
+    for (file_name, path) in test_paths.into_iter() {
+        print!("test {} ... ", path);
+        if ignore(&file_name) {
+            num_ignored += 1;
+            println!("ignored");
+        } else {
+            match run_test(path) {
+                Ok(_) => println!("ok"),
+                Err(_) => {
+                    num_failed += 1;
+                    println!("failed");
+                }
+            }
+        }
+    }
+
+    let result_str = if num_failed > 0 { "failed" } else { "ok" };
+    let num_passed = total - num_failed;
+    println!(
+        "test result: {result_str}. {num_passed} passed; {num_failed} failed; {num_ignored} \
+         ignored",
+    );
+
+    if num_failed == 0 {
+        QEMU_EXIT_HANDLE.exit_success();
+    } else {
+        QEMU_EXIT_HANDLE.exit_failure();
+    }
+}
+
+#[allow(clippy::result_unit_err)]
+pub fn run_test(path: Path) -> Result<(), ()> {
+    match spawn::new_application_task_builder(path, None)
+        .unwrap()
+        .argument(Vec::new())
+        .spawn()
+        .unwrap()
+        .join()
+        .unwrap()
+    {
+        ExitValue::Completed(status) => match status.downcast_ref::<isize>() {
+            Some(0) => Ok(()),
+            _ => Err(()),
+        },
+        ExitValue::Killed(KillReason::Requested) => unreachable!(),
+        ExitValue::Killed(KillReason::Panic(_)) => Err(()),
+        ExitValue::Killed(KillReason::Exception(_)) => Err(()),
+    }
+}
+
+fn ignore(name: &str) -> bool {
+    const IGNORED_TESTS: [&str; 3] = [
+        // `test_libc` requires extra Make commands to run.
+        "test_libc",
+        // `test_panic` panics on success, which isn't easily translatable to
+        // `ExitValue::Completed(0)`.
+        "test_panic",
+        // TODO: Remove
+        // `test_channel` has a bug that causes deadlock.
+        "test_channel",
+    ];
+    for test in IGNORED_TESTS {
+        if name.starts_with(test) {
+            return true;
+        }
+    }
+    false
+}

applications/test_serial_echo/Cargo.toml → applications/serial_echo/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "test_serial_echo"
+name = "serial_echo"
 version = "0.1.0"
 authors = ["Kevin Boos <kevinaboos@gmail.com>"]
 description = "a simple app for testing serial port I/O using higher-level I/O traits"

applications/test_aligned_page_allocation/src/lib.rs

@@ -27,17 +27,15 @@ fn rmain() -> Result<(), &'static str> {
     for num_pages in TEST_SET.into_iter() {
         for alignment in TEST_SET.into_iter() {
             println!("Attempting to allocate {num_pages} pages with alignment of {alignment} 4K pages...");
-            match memory::allocate_pages_deferred(
-                AllocationRequest::AlignedTo { alignment_4k_pages: alignment },
+            let (ap, _action) = memory::allocate_pages_deferred(
+                AllocationRequest::AlignedTo {
+                    alignment_4k_pages: alignment,
+                },
                 num_pages,
-            ) {
-                Ok((ap, _action)) => {
-                    assert_eq!(ap.start().number() % alignment, 0);
-                    assert_eq!(ap.size_in_pages(), num_pages);
-                    println!("    Success: {ap:?}");
-                }
-                Err(e) => println!("    !! FAILURE: {e:?}"),
-            }
+            )?;
+            assert_eq!(ap.start().number() % alignment, 0);
+            assert_eq!(ap.size_in_pages(), num_pages);
+            println!("    Success: {ap:?}");
         }
     }
 

applications/test_block_io/Cargo.toml

@@ -3,6 +3,7 @@ name = "test_block_io"
 version = "0.1.0"
 authors = ["Kevin Boos <kevinaboos@gmail.com>"]
 description = "a simple app for testing IO transfers for block devices"
+edition = "2021"
 
 [dependencies]
 core2 = { version = "0.4.0", default-features = false, features = ["alloc", "nightly"] }