Adds external key-value string data storage to runtime via ExternalDa…

…ta objects in Layers.

Adds --external-data argument to spfs run/shell, --get and --get-all
arguments to spfs info, spfs runtime info, and adds support for
reading the value from an ExternalData using spfs read.

Signed-off-by: David Gilligan-Cook <dcook@imageworks.com>

crates/spfs-cli/cmd-enter/src/cmd_enter.rs

@@ -175,6 +175,15 @@ impl CmdEnter {
             // For example, the monitor may need to run fusermount to clean up
             // fuse filesystems within the runtime before shutting down.
             tracing::debug!("initializing runtime {owned:#?}");
+            tracing::debug!(
+                "runtime stack: {:?}",
+                owned
+                    .status
+                    .stack
+                    .iter_bottom_up()
+                    .map(|d| d.to_string())
+                    .collect::<Vec<_>>()
+            );
             let start_time = Instant::now();
             let render_summary = spfs::initialize_runtime(&mut owned).await?;
             self.report_render_summary(render_summary, start_time.elapsed().as_secs_f64());

crates/spfs-cli/cmd-render/src/cmd_render.rs

@@ -156,7 +156,14 @@ impl CmdRender {
                 &render_summary_reporter as &dyn spfs::storage::fs::RenderReporter,
             ]),
         );
-        let stack = layers.into_iter().map(|l| *l.manifest()).collect();
+        let stack = layers
+            .into_iter()
+            .filter_map(|l| match l.manifest() {
+                None => None,
+                Some(m) => Some(*m),
+            })
+            .collect();
+        tracing::trace!("stack: {:?}", stack);
         renderer
             .render(&stack, self.strategy)
             .await

crates/spfs-cli/common/Cargo.toml

@@ -27,6 +27,7 @@ sentry = { workspace = true, optional = true }
 sentry-miette = { workspace = true, optional = true }
 sentry-tracing = { workspace = true, optional = true }
 serde_json = { version = "1.0.57", optional = true }
+serde_yaml = { workspace = true }
 spfs = { path = "../../spfs" }
 strip-ansi-escapes = { workspace = true, optional = true }
 syslog-tracing = "0.2.0"

crates/spfs-cli/common/src/args.rs

@@ -7,9 +7,10 @@ use std::panic::catch_unwind;
 #[cfg(feature = "sentry")]
 use std::sync::Mutex;
 
-use miette::Error;
+use miette::{Error, IntoDiagnostic, Result, WrapErr};
 #[cfg(feature = "sentry")]
 use once_cell::sync::OnceCell;
+use spfs::io::Pluralize;
 use spfs::storage::LocalRepository;
 use tracing_subscriber::prelude::*;
 
@@ -416,6 +417,62 @@ impl Logging {
     }
 }
 
+/// Command line flags for viewing external data in a runtime
+#[derive(Debug, Clone, clap::Args)]
+pub struct ExternalDataViewing {
+    /// Output the data value for the given external data key(s) from
+    /// the active runtime. Each value is printed on its own line
+    /// without its key.
+    #[clap(long, alias = "external_data")]
+    pub get: Option<Vec<String>>,
+
+    /// Output all the external data keys and values from the active
+    /// runtime as a yaml dictionary
+    #[clap(long, alias = "all_external_data")]
+    pub get_all: bool,
+}
+
+impl ExternalDataViewing {
+    /// Display external data values based on the command line arguments
+    pub async fn print_data(&self, runtime: &spfs::runtime::Runtime) -> Result<()> {
+        if self.get_all {
+            let data = runtime.all_external_data().await?;
+            let keys = data
+                .keys()
+                .map(ToString::to_string)
+                .collect::<Vec<String>>();
+            let num_keys = keys.len();
+            tracing::debug!(
+                "{num_keys} external data {}: {}",
+                "key".pluralize(num_keys),
+                keys.join(", ")
+            );
+            println!(
+                "{}",
+                serde_yaml::to_string(&data)
+                    .into_diagnostic()
+                    .wrap_err("Failed to generate yaml output")?
+            );
+        } else if let Some(keys) = &self.get {
+            tracing::debug!("--get these keys: {}", keys.join(", "));
+            for key in keys.iter() {
+                match runtime.external_data(key).await? {
+                    Some(value) => {
+                        tracing::debug!("{key} = {value}");
+                        println!("{value}");
+                    }
+                    None => {
+                        tracing::warn!("No external data stored under: {key}");
+                        println!();
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
 /// Trait all spfs cli command parsers must implement to provide the
 /// name of the spfs command that has been parsed. This method will be
 /// called when configuring sentry.

crates/spfs-cli/common/src/lib.rs

@@ -11,6 +11,6 @@ pub mod __private {
     pub use {libc, spfs};
 }
 
-pub use args::{capture_if_relevant, CommandName, Logging, Render, Sync};
+pub use args::{capture_if_relevant, CommandName, ExternalDataViewing, Logging, Render, Sync};
 #[cfg(feature = "sentry")]
 pub use args::{configure_sentry, shutdown_sentry};

crates/spfs-cli/main/Cargo.toml

@@ -32,6 +32,7 @@ nix = { workspace = true }
 number_prefix = "*"                                                           # we hope to match versions with indicatif
 relative-path = "1.3"
 serde_json = { workspace = true }
+serde_yaml = { workspace = true }
 spfs = { path = "../../spfs" }
 spfs-cli-common = { path = "../common" }
 strum = { workspace = true, features = ["derive"] }
@@ -52,3 +53,7 @@ features = [
     "Win32_System_SystemInformation",
     "Win32_System_Threading",
 ]
+
+[dev-dependencies]
+rstest = { workspace = true }
+tempfile = { workspace = true }

crates/spfs-cli/main/src/cmd_info.rs

@@ -7,6 +7,7 @@ use colored::*;
 use miette::Result;
 use spfs::env::SPFS_DIR;
 use spfs::find_path::ObjectPathEntry;
+use spfs::graph::ExternalData;
 use spfs::io::{self, DigestFormat, Pluralize};
 use spfs::prelude::*;
 use spfs::{self};
@@ -18,6 +19,9 @@ pub struct CmdInfo {
     #[clap(flatten)]
     logging: cli::Logging,
 
+    #[clap(flatten)]
+    external_data: cli::ExternalDataViewing,
+
     /// Lists file sizes in human readable format
     #[clap(long, short = 'H')]
     human_readable: bool,
@@ -129,8 +133,20 @@ impl CmdInfo {
                 println!(
                     " {} {}",
                     "manifest:".bright_blue(),
-                    self.format_digest(*obj.manifest(), repo).await?
+                    match obj.manifest() {
+                        None => "None".to_string(),
+                        Some(manifest_digest) => self.format_digest(*manifest_digest, repo).await?,
+                    }
                 );
+
+                if let Some(data) = obj.external_data() {
+                    let external_data: ExternalData = data.into();
+                    println!(" {}", "external data:".bright_blue());
+                    println!("  {} {}", "key:".bright_blue(), external_data.key());
+                    println!("  {} {:?}", "value:".bright_blue(), external_data.value());
+                } else {
+                    println!(" {} none", "external data:".bright_blue());
+                }
             }
 
             Enum::Manifest(obj) => {
@@ -186,6 +202,10 @@ impl CmdInfo {
     async fn print_global_info(&self, repo: &spfs::storage::RepositoryHandle) -> Result<()> {
         let runtime = spfs::active_runtime().await?;
 
+        if self.external_data.get_all || self.external_data.get.is_some() {
+            return self.external_data.print_data(&runtime).await;
+        }
+
         println!("{}:", "Active Runtime".green());
         println!(" {}: {}", "id".bright_blue(), runtime.name());
         println!(" {}: {}", "editable".bright_blue(), runtime.status.editable);

crates/spfs-cli/main/src/cmd_run.rs

@@ -2,15 +2,112 @@
 // SPDX-License-Identifier: Apache-2.0
 // https://github.com/imageworks/spk
 
+use std::collections::BTreeMap;
 use std::ffi::OsString;
+use std::io;
 use std::time::Instant;
 
 use clap::{ArgGroup, Args};
-use miette::{Context, Result};
+use miette::{miette, Context, IntoDiagnostic, Result};
+use spfs::graph::object::EncodingFormat;
 use spfs::prelude::*;
+use spfs::runtime::KeyValuePair;
+use spfs::storage::FromConfig;
 use spfs::tracking::EnvSpec;
 use spfs_cli_common as cli;
 
+#[cfg(test)]
+#[path = "./cmd_run_test.rs"]
+mod cmd_run_test;
+#[cfg(test)]
+#[path = "./fixtures.rs"]
+mod fixtures;
+
+#[derive(Args, Clone, Debug)]
+pub struct ExternalData {
+    /// Adds extra external key-value string data to the new runtime.
+    ///
+    /// This allows external processes to store arbitrary data in the
+    /// runtimes they create. This is most useful with durable runtimes.
+    /// The data can be retrieved by running `spfs runtime info` or
+    /// `spfs info` and using the `--get <KEY>` or `--get-all` options
+    ///
+    /// External data is specified as key-value string pairs separated
+    /// by either an equals sign or colon (--external-data name=value
+    /// --external-data other:value). Multiple pairs of external data
+    /// can also be specified at once in yaml or json format
+    /// (--external-data '{name: value, other: value}').
+    ///
+    /// External data can also be given in a json or yaml file, by
+    /// using the `--external-data-file <FILE>` argument. If given,
+    /// `--external-data` arguments will supersede anything given in
+    /// external data files.
+    ///
+    /// If the same key is used more than once, the last key-value pair
+    /// will override the earlier values for the same key.
+    #[clap(long, value_name = "KEY:VALUE")]
+    pub external_data: Vec<String>,
+
+    /// Specify external extra key-value data from a json or yaml file
+    /// (see --external-data)
+    #[clap(long)]
+    pub external_data_file: Vec<std::path::PathBuf>,
+}
+
+impl ExternalData {
+    /// Returns a list of external data key-value pairs gathered from
+    /// all the external data related command line arguments. The same
+    /// keys, and values, can appear multiple times in the list if
+    /// specified multiple times in various command line arguments.
+    pub fn get_data(&self) -> Result<Vec<KeyValuePair>> {
+        let mut data: Vec<KeyValuePair> = Vec::new();
+
+        for filename in self.external_data_file.iter() {
+            let reader: Box<dyn io::Read> =
+                if Ok("-".to_string()) == filename.clone().into_os_string().into_string() {
+                    // Treat '-' as "read from stdin"
+                    Box::new(io::stdin())
+                } else {
+                    Box::new(
+                        std::fs::File::open(filename)
+                            .into_diagnostic()
+                            .wrap_err(format!("Failed to open external data file: {filename:?}"))?,
+                    )
+                };
+            let external_data: BTreeMap<String, String> = serde_yaml::from_reader(reader)
+                .into_diagnostic()
+                .wrap_err(format!(
+                    "Failed to parse as external data key-value pairs: {filename:?}"
+                ))?;
+            data.extend(external_data);
+        }
+
+        for pair in self.external_data.iter() {
+            let pair = pair.trim();
+            if pair.starts_with('{') {
+                let given: BTreeMap<String, String> = serde_yaml::from_str(pair)
+                    .into_diagnostic()
+                    .wrap_err("--external-data value looked like yaml, but could not be parsed")?;
+                data.extend(given);
+                continue;
+            }
+
+            let (name, value) = pair
+                .split_once('=')
+                .or_else(|| pair.split_once(':'))
+                .ok_or_else(|| {
+                    miette!(
+                        "Invalid option: -external-data {pair} (should be in the form name=value)"
+                    )
+                })?;
+
+            data.push((name.to_string(), value.to_string()));
+        }
+
+        Ok(data)
+    }
+}
+
 /// Run a program in a configured spfs environment
 #[derive(Debug, Args)]
 #[clap(group(
@@ -51,6 +148,9 @@ pub struct CmdRun {
     #[clap(long, value_name = "RUNTIME_NAME")]
     pub rerun: Option<String>,
 
+    #[clap(flatten)]
+    pub external_data: ExternalData,
+
     /// The tag or id of the desired runtime
     ///
     /// Use '-' to request an empty environment
@@ -146,6 +246,26 @@ impl CmdRun {
                 );
             }
 
+            let data = self.external_data.get_data()?;
+            if !data.is_empty() {
+                if config.storage.encoding_format == EncodingFormat::Legacy {
+                    return Err(spfs::Error::String(
+                        "Cannot use '--external-data' when spfs is configured to use the 'Legacy' encoding format".to_string(),
+                    )
+                    .into());
+                }
+
+                // These are added in reverse order so that the ones
+                // specified later on the command line will take precedence.
+                for (key, value) in data.into_iter().rev() {
+                    tracing::trace!("ex data being added: {key}: {value}");
+                    runtime
+                        .add_external_data(key, value, config.filesystem.external_data_size_limit)
+                        .await?;
+                }
+                tracing::trace!(" with external data: {:?}", runtime);
+            }
+
             let start_time = Instant::now();
             runtime.config.mount_backend = config.filesystem.backend;
             runtime.config.secondary_repositories = config.get_secondary_runtime_repositories();