time runtime suite executions

Developers of synthetic tests are required to specify a timeout to
ensure that their test does not run longer than specified. Previously,
the ouput of the rebot command was used as proxy for this command.
Currently, there are some bugs in the rebot command in v7. This causes
the runtimes to be computed incorrectly.

We know time the runtime of each attempt instead.

src/results.rs

@@ -81,6 +81,7 @@ impl WritePiggybackSection for SuiteExecutionReport {
 pub struct AttemptReport {
     pub index: usize,
     pub outcome: AttemptOutcome,
+    pub runtime: i64,
 }
 
 #[derive(PartialEq, Debug, Serialize)]

src/suites.rs

@@ -6,6 +6,7 @@ use crate::session::{RunSpec, Session};
 use crate::termination::{Cancelled, Outcome};
 use anyhow::Context;
 use camino::{Utf8Path, Utf8PathBuf};
+use chrono::Utc;
 use log::{error, info};
 use tokio_util::sync::CancellationToken;
 
@@ -24,6 +25,7 @@ pub fn run_attempts_with_rebot(
     for attempt in robot.attempts(output_directory) {
         info!("Suite {id}: running attempt {}", attempt.index);
         let attempt_index = attempt.index;
+        let starttime = Utc::now();
         let (outcome, output_path) = run_attempt(
             id,
             environment,
@@ -33,10 +35,12 @@ pub fn run_attempts_with_rebot(
             cancellation_token,
             output_directory,
         )?;
+        let endtime = Utc::now();
         let success = matches!(&outcome, &AttemptOutcome::AllTestsPassed);
         attempt_reports.push(AttemptReport {
             index: attempt_index,
             outcome,
+            runtime: (endtime - starttime).num_seconds(),
         });
         if let Some(output_path) = output_path {
             output_paths.push(output_path);

tests/test_suite_run.rs

@@ -2,7 +2,7 @@ use anyhow::Result as AnyhowResult;
 use camino::Utf8PathBuf;
 use robotmk::config::RetryStrategy;
 use robotmk::environment::{Environment, SystemEnvironment};
-use robotmk::results::{AttemptOutcome, AttemptReport};
+use robotmk::results::AttemptOutcome;
 use robotmk::rf::robot::Robot;
 use robotmk::session::{CurrentSession, Session};
 use robotmk::suites::run_attempts_with_rebot;
@@ -27,13 +27,10 @@ fn test_rebot_run() -> AnyhowResult<()> {
         &CancellationToken::default(),
         &test_dir,
     )?;
-    assert_eq!(
-        attempt_reports,
-        &[AttemptReport {
-            index: 1,
-            outcome: AttemptOutcome::AllTestsPassed,
-        }]
-    );
+    assert_eq!(attempt_reports.len(), 1);
+    let attempt_report = &attempt_reports[0];
+    assert_eq!(attempt_report.index, 1);
+    assert_eq!(attempt_report.outcome, AttemptOutcome::AllTestsPassed);
     assert!(rebot.is_some());
     Ok(())
 }
@@ -56,13 +53,10 @@ fn test_timeout_process() -> AnyhowResult<()> {
         &CancellationToken::default(),
         &test_dir,
     )?;
+    assert_eq!(attempt_reports.len(), 1);
+    let attempt_report = &attempt_reports[0];
+    assert_eq!(attempt_report.index, 1);
+    assert_eq!(attempt_report.outcome, AttemptOutcome::TimedOut);
     assert!(rebot.is_none());
-    assert_eq!(
-        attempt_reports,
-        &[AttemptReport {
-            index: 1,
-            outcome: AttemptOutcome::TimedOut,
-        }]
-    );
     Ok(())
 }