From cf5af6aadabb19e0bb6748204a442a88e9fff9f5 Mon Sep 17 00:00:00 2001
From: Charly Castes <charly.castes@epfl.ch>
Date: Mon, 9 Dec 2024 23:48:41 +0100
Subject: [PATCH] Runner: add option for expected output in tests

This commit adds a new test parameter, 'expect', which takes a substring
that should be present in the output of the test. This is useful to
ensure certain things happen. For instance Linux can successfully boot
on a multi-core machine even if a hart is not responding, which this
patch we can check that all harts booted properly.
---
 miralis.toml          |  1 +
 runner/src/project.rs |  2 ++
 runner/src/test.rs    | 37 ++++++++++++++++++++++++++++++++++---
 3 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/miralis.toml b/miralis.toml
index 6b5aa359..3449ae3d 100644
--- a/miralis.toml
+++ b/miralis.toml
@@ -180,6 +180,7 @@ description = "Run Linux and exit as soon as it reaches userspace on a sifive u5
 firmware = "linux"
 config = "qemu-virt-2harts"
 description = "Run linux with two cores, expecting it to boot with both"
+expect = "smp: Brought up 1 node, 2 CPUs"
 
 ## ———————————————————————————— Testing Policies ———————————————————————————— ##
 
diff --git a/runner/src/project.rs b/runner/src/project.rs
index b36392df..2ce75862 100644
--- a/runner/src/project.rs
+++ b/runner/src/project.rs
@@ -30,4 +30,6 @@ pub struct Test {
     pub description: Option<String>,
     pub firmware: Option<String>,
     pub payload: Option<String>,
+    /// An expected string from the output of the test
+    pub expect: Option<String>,
 }
diff --git a/runner/src/test.rs b/runner/src/test.rs
index 58b33499..19ff85bf 100644
--- a/runner/src/test.rs
+++ b/runner/src/test.rs
@@ -2,8 +2,9 @@
 
 use std::collections::HashMap;
 use std::fs;
+use std::io::Read;
 use std::path::PathBuf;
-use std::process::ExitCode;
+use std::process::{ExitCode, Stdio};
 
 use crate::artifacts::{build_target, prepare_firmware_artifact, Target};
 use crate::config::{read_config, Config, Platforms};
@@ -170,6 +171,7 @@ pub fn run_one_test(test: &Test, test_name: &str, cfg: &Config) -> Result<(), Op
         return Err(None);
     };
 
+    // Prepare the command to run
     let cmd = match cfg.platform.name.unwrap_or(Platforms::QemuVirt) {
         Platforms::QemuVirt => {
             get_qemu_cmd(cfg, miralis, firmware, test.payload.as_ref(), false, false)
@@ -194,9 +196,38 @@ pub fn run_one_test(test: &Test, test_name: &str, cfg: &Config) -> Result<(), Op
             .join(" ")
     );
 
-    let exit_status = cmd.status().expect("Failed to run");
+    // Then execute the test and check for the success criteria
+    //
+    // For some tests we require a substring to be present in the output, in those cases we do some
+    // aditionnal work on top of checking the exit status.
+    let mut succeeded = true;
+    let exit_status = if let Some(expected) = &test.expect {
+        // We need to get the output of the child, we create a pipe for that purpose
+        cmd.stdout(Stdio::piped());
+        let mut child = cmd.spawn().expect("Failed to spawn command");
+        let pipe = child
+            .stdout
+            .as_mut()
+            .expect("Could not read child process output");
+        let mut buff = Vec::new();
+        pipe.read_to_end(&mut buff)
+            .expect("Failed to read output from child process");
+        let exit_status = child.wait().expect("Failed to wait for child process");
+
+        // We got the exit status, now also check for the expected pattern
+        let buff = String::from_utf8_lossy(&buff);
+        if !buff.contains(expected) {
+            log::error!("Could not find '{}' in the test output", expected);
+            succeeded = false;
+        }
+
+        exit_status
+    } else {
+        // log::warn!("Test :)");
+        cmd.status().expect("Failed to run")
+    };
 
-    if !exit_status.success() {
+    if !exit_status.success() || !succeeded {
         let cmd_str = format!(
             "{} {}",
             cmd.get_program().to_str().unwrap(),