work on integration testing framework (#41)

Author: koehlma

.cspell.json

@@ -38,6 +38,7 @@
     "Chunker",
     "cmdline",
     "composability",
+    "cpus",
     "discoverability",
     "Discoverability",
     "editenv",

crates/reportify/src/lib.rs

@@ -276,6 +276,7 @@ use std::{
     any::Any,
     error::Error as StdError,
     fmt::{Debug, Display},
+    future::Future,
 };
 
 use backtrace::BacktraceImpl;
@@ -327,7 +328,7 @@ impl<E: 'static + StdError + Send> Error for E {
     }
 }
 
-trait AnyReport {
+trait AnyReport: Send {
     fn error(&self) -> &dyn Error;
 
     fn meta(&self) -> &ReportMeta;

crates/rugpi-bakery/src/bake/image.rs

@@ -25,7 +25,7 @@ use crate::{
         rpi_uboot::initialize_uboot, Target,
     },
     project::images::{self, grub_efi_image_layout, pi_image_layout, ImageConfig, ImageLayout},
-    utils::prelude::*,
+    utils::{caching::mtime, prelude::*},
     BakeryResult,
 };
 
@@ -37,6 +37,15 @@ pub fn make_image(config: &ImageConfig, src: &Path, image: &Path) -> BakeryResul
         fs::create_dir_all(parent).ok();
     }
 
+    if image.exists() {
+        let src_mtime = mtime(src).whatever("unable to get image mtime")?;
+        let image_mtime = mtime(image).whatever("unable to get image mtime")?;
+        if src_mtime <= image_mtime {
+            info!("Image is newer than sources.");
+            return Ok(());
+        }
+    }
+
     // Initialize system root directory from provided TAR file.
     info!("Extracting layer.");
     run!(["tar", "-xf", src, "-C", &bundle_dir]).whatever("unable to extract layer")?;

crates/rugpi-bakery/src/main.rs

@@ -1,7 +1,7 @@
 use std::{
     collections::HashMap,
     convert::Infallible,
-    ffi::{CStr, CString},
+    ffi::{CStr, CString, OsStr},
     fs,
     path::{Path, PathBuf},
 };
@@ -15,6 +15,7 @@ use project::{
 use reportify::{bail, Report, ResultExt};
 use rugpi_common::fsutils::copy_recursive;
 use serde::Deserialize;
+use test::RugpiTestError;
 
 pub mod bake;
 pub mod project;
@@ -93,7 +94,7 @@ pub enum BakeCommand {
 /// The `test` command.
 #[derive(Debug, Parser)]
 pub struct TestCommand {
-    case: PathBuf,
+    workflows: Vec<String>,
 }
 
 /// The `bake` command.
@@ -233,11 +234,38 @@ fn main() -> BakeryResult<()> {
             }
         }
         Command::Test(test_command) => {
+            let project = load_project(&args)?;
             tokio::runtime::Builder::new_multi_thread()
                 .enable_all()
                 .build()
                 .unwrap()
-                .block_on(test::main(&test_command.case))
+                .block_on(async move {
+                    let mut workflows = Vec::new();
+                    if test_command.workflows.is_empty() {
+                        let mut read_dir = tokio::fs::read_dir(project.dir.join("tests"))
+                            .await
+                            .whatever("unable to scan for test workflows")?;
+                        while let Some(entry) = read_dir
+                            .next_entry()
+                            .await
+                            .whatever("unable to read entry")?
+                        {
+                            let path = entry.path();
+                            if path.extension() == Some(OsStr::new("toml")) {
+                                workflows.push(path);
+                            }
+                        }
+                    } else {
+                        for name in &test_command.workflows {
+                            workflows
+                                .push(project.dir.join("tests").join(name).with_extension("toml"));
+                        }
+                    };
+                    for workflow in workflows {
+                        test::main(&project, &workflow).await?;
+                    }
+                    <Result<(), Report<RugpiTestError>>>::Ok(())
+                })
                 .whatever("unable to run test")?;
         }
     }

crates/rugpi-bakery/src/project/mod.rs

@@ -1,15 +1,14 @@
 //! In-memory representation of Rugpi Bakery projects.
 
 use std::{
-    cell::OnceCell,
     path::{Path, PathBuf},
-    sync::Arc,
+    sync::{Arc, OnceLock},
 };
 
 use reportify::ResultExt;
 
 use self::{config::BakeryConfig, library::Library, repositories::ProjectRepositories};
-use crate::{utils::prelude::*, BakeryResult};
+use crate::BakeryResult;
 
 pub mod config;
 pub mod images;
@@ -19,7 +18,7 @@ pub mod recipes;
 pub mod repositories;
 
 /// A project.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 #[non_exhaustive]
 pub struct Project {
     /// The configuration of the project.
@@ -33,30 +32,37 @@ pub struct Project {
 impl Project {
     /// The repositories of the project.
     pub fn repositories(&self) -> BakeryResult<&Arc<ProjectRepositories>> {
-        self.lazy
-            .repositories
-            .try_get_or_init(|| ProjectRepositories::load(self).map(Arc::new))
-            .whatever("loading repositories")
+        if let Some(repositories) = self.lazy.repositories.get() {
+            return Ok(repositories);
+        }
+        let repositories = ProjectRepositories::load(self)
+            .map(Arc::new)
+            .whatever("loading repositories")?;
+        let _ = self.lazy.repositories.set(repositories);
+        Ok(self.lazy.repositories.get().unwrap())
     }
 
     /// The library of the project.
     pub fn library(&self) -> BakeryResult<&Arc<Library>> {
-        self.lazy.library.try_get_or_init(|| {
-            let repositories = self.repositories()?.clone();
-            Library::load(repositories)
-                .map(Arc::new)
-                .whatever("loading library")
-        })
+        if let Some(library) = self.lazy.library.get() {
+            return Ok(library);
+        }
+        let repositories = self.repositories()?.clone();
+        let library = Library::load(repositories)
+            .map(Arc::new)
+            .whatever("loading library")?;
+        let _ = self.lazy.library.set(library);
+        Ok(self.lazy.library.get().unwrap())
     }
 }
 
 /// Lazily initialized fields of [`Project`].
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 struct ProjectLazy {
     /// The repositories of the project.
-    repositories: OnceCell<Arc<ProjectRepositories>>,
+    repositories: OnceLock<Arc<ProjectRepositories>>,
     /// The library of the project.
-    library: OnceCell<Arc<Library>>,
+    library: OnceLock<Arc<Library>>,
 }
 
 /// Project loader.

crates/rugpi-bakery/src/test/mod.rs

@@ -1,61 +1,77 @@
 use std::{path::Path, time::Duration};
 
-use case::{TestCase, TestStep};
 use reportify::{bail, Report, ResultExt};
 use rugpi_cli::info;
-use tokio::fs;
+use tokio::{fs, task::spawn_blocking};
+use workflow::{TestStep, TestWorkflow};
+
+use crate::{bake, project::Project};
 
-pub mod case;
 pub mod qemu;
+pub mod workflow;
 
 reportify::new_whatever_type! {
     RugpiTestError
 }
 
 pub type RugpiTestResult<T> = Result<T, Report<RugpiTestError>>;
 
-pub async fn main(case: &Path) -> RugpiTestResult<()> {
-    let case = toml::from_str::<TestCase>(
-        &fs::read_to_string(&case)
+pub async fn main(project: &Project, workflow: &Path) -> RugpiTestResult<()> {
+    let workflow = toml::from_str::<TestWorkflow>(
+        &fs::read_to_string(&workflow)
             .await
-            .whatever("unable to read test case")?,
+            .whatever("unable to read test workflow")?,
     )
-    .whatever("unable to parse test case")?;
-
-    let vm = qemu::start(&case.vm).await?;
-
-    info!("VM started");
-
-    for step in &case.steps {
-        match step {
-            case::TestStep::Reboot => todo!(),
-            case::TestStep::Copy { .. } => todo!(),
-            case::TestStep::Run {
-                script,
-                stdin,
-                may_fail,
-            } => {
-                info!("running script");
-                vm.wait_for_ssh()
-                    .await
-                    .whatever("unable to connect to VM via SSH")?;
-                if let Err(report) = vm
-                    .run_script(script, stdin.as_ref().map(|p| p.as_ref()))
-                    .await
-                    .whatever::<RugpiTestError, _>("unable to run script")
-                {
-                    if may_fail.unwrap_or(false) {
-                        eprintln!("ignoring error while executing script:\n{report:?}");
-                    } else {
-                        bail!("error during test")
+    .whatever("unable to parse test workflow")?;
+
+    for system in &workflow.systems {
+        let output = Path::new("build/images")
+            .join(&system.disk_image)
+            .with_extension("img");
+        let project = project.clone();
+        let disk_image = system.disk_image.clone();
+        {
+            let output = output.clone();
+            spawn_blocking(move || bake::bake_image(&project, &disk_image, &output))
+                .await
+                .whatever("error baking image")?
+                .whatever("error baking image")?;
+        }
+
+        let vm = qemu::start(&output.to_string_lossy(), system).await?;
+
+        info!("VM started");
+
+        for step in &workflow.steps {
+            match step {
+                workflow::TestStep::Run {
+                    script,
+                    stdin,
+                    may_fail,
+                } => {
+                    info!("running script");
+                    vm.wait_for_ssh()
+                        .await
+                        .whatever("unable to connect to VM via SSH")?;
+                    if let Err(report) = vm
+                        .run_script(script, stdin.as_ref().map(|p| p.as_ref()))
+                        .await
+                        .whatever::<RugpiTestError, _>("unable to run script")
+                    {
+                        if may_fail.unwrap_or(false) {
+                            eprintln!("ignoring error while executing script:\n{report:?}");
+                        } else {
+                            bail!("error during test")
+                        }
                     }
                 }
-            }
-            TestStep::Wait { duration_secs } => {
-                info!("waiting for {duration_secs} seconds");
-                tokio::time::sleep(Duration::from_secs_f64(*duration_secs)).await;
+                TestStep::Wait { duration } => {
+                    info!("waiting for {duration} seconds");
+                    tokio::time::sleep(Duration::from_secs_f64(*duration)).await;
+                }
             }
         }
     }
+
     Ok(())
 }

crates/rugpi-bakery/src/test/qemu.rs

@@ -18,15 +18,15 @@ use tokio::{
     time,
 };
 
-use super::{case::VmConfig, RugpiTestResult};
+use super::{workflow::TestSystemConfig, RugpiTestResult};
 
 pub struct Vm {
     #[expect(dead_code, reason = "not currently used")]
     child: Child,
     ssh_session: Mutex<Option<Handle<SshHandler>>>,
     sftp_session: Mutex<Option<SftpSession>>,
     #[expect(dead_code, reason = "not currently used")]
-    vm_config: VmConfig,
+    vm_config: TestSystemConfig,
     private_key: Arc<PrivateKey>,
 }
 
@@ -158,20 +158,20 @@ impl Vm {
     }
 }
 
-pub async fn start(config: &VmConfig) -> RugpiTestResult<Vm> {
-    let private_key = load_secret_key(&config.private_key, None)
+pub async fn start(image_file: &str, config: &TestSystemConfig) -> RugpiTestResult<Vm> {
+    let private_key = load_secret_key(&config.ssh.private_key, None)
         .whatever("unable to load private SSH key")
-        .with_info(|_| format!("path: {:?}", config.private_key))?;
+        .with_info(|_| format!("path: {:?}", config.ssh.private_key))?;
     fs::create_dir_all(".rugpi/")
         .await
         .whatever("unable to create .rugpi directory")?;
     if !Command::new("qemu-img")
         .args(&["create", "-f", "qcow2", "-F", "raw", "-o"])
-        .arg(format!(
-            "backing_file=../{}",
-            config.image.to_string_lossy()
-        ))
-        .args(&[".rugpi/vm-image.img", "48G"])
+        .arg(format!("backing_file=../{}", image_file))
+        .args(&[
+            ".rugpi/vm-image.img",
+            config.disk_size.as_deref().unwrap_or("40G"),
+        ])
         .spawn()
         .whatever("unable to create VM image")?
         .wait()
@@ -207,30 +207,22 @@ pub async fn start(config: &VmConfig) -> RugpiTestResult<Vm> {
     ]);
     command
         .kill_on_drop(true)
-        .stdout(if config.stdout.is_some() {
-            Stdio::piped()
-        } else {
-            Stdio::null()
-        })
-        .stderr(if config.stderr.is_some() {
-            Stdio::piped()
-        } else {
-            Stdio::null()
-        })
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
         .stdin(Stdio::null());
     let mut child = command.spawn().whatever("unable to spawn Qemu")?;
-    if let Some(stdout) = &config.stdout {
+    if let Some(stdout) = Some("build/vm-stdout.log") {
         let mut stdout_log = fs::File::create(stdout)
             .await
             .whatever("unable to create stdout log file")?;
         let mut stdout = child.stdout.take().expect("we used Stdio::piped");
         tokio::spawn(async move { io::copy(&mut stdout, &mut stdout_log).await });
     }
-    if let Some(stderr) = &config.stderr {
+    if let Some(stderr) = Some("build/vm-stderr.log") {
         let mut stderr_log = fs::File::create(stderr)
             .await
             .whatever("unable to create stderr log file")?;
-        let mut stderr = child.stdout.take().expect("we used Stdio::piped");
+        let mut stderr = child.stderr.take().expect("we used Stdio::piped");
         tokio::spawn(async move { io::copy(&mut stderr, &mut stderr_log).await });
     }
     // We give Qemu some time to start before checking it's exit status.