Add live layer support to spfs and spk (#814)

* Adds live layer support to spfs and spk

Adds LiveLayer, LiveLayerFile, LiveLayerContents, LiveLayerApiVersion.
BindMounts and parsing, loading, validation, preparation, bind mounting 
and unmounting methods.

Signed-off-by: David Gilligan-Cook <dcook@imageworks.com>

crates/spfs-cli/cmd-clean/src/cmd_clean.rs

@@ -121,11 +121,10 @@ impl CommandName for CmdClean {
 impl CmdClean {
     pub async fn run(&mut self, config: &spfs::Config) -> Result<i32> {
         let repo = spfs::config::open_repository_from_string(config, self.remote.as_ref()).await?;
-
-        tracing::info!("clean called");
+        tracing::debug!("spfs clean command called");
 
         if let Some(runtime_name) = &self.remove_durable {
-            tracing::info!("durable: rt name: {}", runtime_name);
+            tracing::debug!("durable rt name: {}", runtime_name);
             // Remove the durable path associated with the runtime,if
             // there is one. This uses the runtime_storage option
             // because the repo name is not available from the spfs

crates/spfs-cli/main/src/cmd_ls.rs

@@ -57,7 +57,9 @@ impl CmdLs {
                 self.username = tag.username_without_org().to_string();
                 self.last_modified = tag.time.format("%b %e %H:%M").to_string();
             }
-            EnvSpecItem::PartialDigest(_) | EnvSpecItem::Digest(_) => (),
+            EnvSpecItem::PartialDigest(_)
+            | EnvSpecItem::Digest(_)
+            | EnvSpecItem::LiveLayerFile(_) => (),
         }
 
         let item = repo.read_ref(&self.reference.to_string()).await?;

crates/spfs-cli/main/src/cmd_run.rs

@@ -95,6 +95,10 @@ impl CmdRun {
                 runtime.reinit_for_reuse_and_save_to_storage().await?;
             }
 
+            // TODO: there's nothing to change or clear any extra
+            // mounts made the last time this durable runtime was run.
+            // Currently, will use the extra mounts already in the runtime.
+
             let start_time = Instant::now();
             let origin = config.get_remote("origin").await?;
             let references_to_sync = EnvSpec::from_iter(runtime.status.stack.iter().copied());
@@ -107,20 +111,40 @@ impl CmdRun {
 
             self.exec_runtime_command(&mut runtime, &start_time).await
         } else if let Some(reference) = &self.reference {
+            let live_layers = reference.load_live_layers()?;
+            if !live_layers.is_empty() {
+                tracing::debug!("with live layers: {live_layers:?}");
+            };
+
             // Make a new empty runtime
             let mut runtime = match &self.runtime_name {
                 Some(name) => {
                     runtimes
-                        .create_named_runtime(name, self.keep_runtime)
+                        .create_named_runtime(name, self.keep_runtime, live_layers)
+                        .await?
+                }
+                None => {
+                    runtimes
+                        .create_runtime(self.keep_runtime, live_layers)
                         .await?
                 }
-                None => runtimes.create_runtime(self.keep_runtime).await?,
             };
-            tracing::debug!(
-                "created new runtime: {} [keep={}]",
-                runtime.name(),
-                self.keep_runtime
-            );
+
+            if self.keep_runtime && self.runtime_name.is_none() {
+                // User wants a durable runtime but has not named it,
+                // which means it will get uuid name. Want to make
+                // them aware of this and how to name it next time.
+                tracing::warn!(
+                    "created new durable runtime without naming it. You can use --runtime-name NAME to give the runtime a name. This one will be called: {}",
+                    runtime.name()
+                );
+            } else {
+                tracing::debug!(
+                    "created new runtime: {} [keep={}]",
+                    runtime.name(),
+                    self.keep_runtime
+                );
+            }
 
             let start_time = Instant::now();
             runtime.config.mount_backend = config.filesystem.backend;

crates/spfs/Cargo.toml

@@ -54,6 +54,7 @@ semver = "1.0"
 sentry = { workspace = true, optional = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
+serde_yaml = { workspace = true }
 serde_qs = "0.10.1"
 spfs-encoding = { path = "../spfs-encoding" }
 strum = { workspace = true, features = ["derive"] }

crates/spfs/src/check.rs

@@ -159,6 +159,11 @@ where
                 .check_tag_spec(tag_spec)
                 .await
                 .map(CheckEnvItemResult::Tag)?,
+            tracking::EnvSpecItem::LiveLayerFile(_) => {
+                // These items do not need checking, they can be ignored.
+                // They do no represent spfs objects in a repo.
+                CheckEnvItemResult::Object(CheckObjectResult::Ignorable)
+            }
         };
         Ok(res)
     }
@@ -768,6 +773,9 @@ impl CheckTagResult {
 pub enum CheckObjectResult {
     /// The object was already checked in this session
     Duplicate,
+    /// The object can be ignored, it does not need checking, it has
+    /// no representation in the database.
+    Ignorable,
     /// The object was found to be missing from the database
     Missing(encoding::Digest),
     Platform(CheckPlatformResult),
@@ -784,6 +792,7 @@ impl CheckObjectResult {
     fn set_repaired(&mut self) {
         match self {
             CheckObjectResult::Duplicate => (),
+            CheckObjectResult::Ignorable => (),
             CheckObjectResult::Missing(_) => (),
             CheckObjectResult::Platform(r) => r.set_repaired(),
             CheckObjectResult::Layer(r) => r.set_repaired(),
@@ -798,6 +807,7 @@ impl CheckObjectResult {
         use CheckObjectResult::*;
         match self {
             Duplicate => CheckSummary::default(),
+            Ignorable => CheckSummary::default(),
             Missing(digest) => CheckSummary {
                 missing_objects: Some(*digest).into_iter().collect(),
                 ..Default::default()

crates/spfs/src/env.rs

@@ -11,6 +11,7 @@ use super::runtime;
 use crate::{which, Error, Result};
 
 pub const SPFS_DIR: &str = "/spfs";
+pub const SPFS_DIR_PREFIX: &str = "/spfs/";
 
 const NONE: Option<&str> = None;
 
@@ -412,6 +413,87 @@ where
         rt.ensure_required_directories().await
     }
 
+    async fn mount_live_layers(&self, rt: &runtime::Runtime) -> Result<()> {
+        // Mounts the bind mounts from the any live layers in the runtime the top of paths
+        // inside /spfs
+        //
+        // It requires the mount destinations to exist under
+        // /spfs/. If they do not, the mount commands will error. The
+        // mount destinations are either provided by one of the layers
+        // in the runtime, or by an earlier call to
+        // ensure_extra_bind_mount_locations_exist() made in
+        // initialize_runtime()
+        let live_layers = rt.live_layers();
+        if !live_layers.is_empty() {
+            tracing::debug!("mounting the extra bind mounts over the {SPFS_DIR} filesystem ...");
+            let mount = super::resolve::which("mount").unwrap_or_else(|| "/usr/bin/mount".into());
+
+            for layer in live_layers {
+                let injection_mounts = layer.bind_mounts();
+
+                for extra_mount in injection_mounts {
+                    let dest = if extra_mount.dest.starts_with(SPFS_DIR_PREFIX) {
+                        PathBuf::from(extra_mount.dest.clone())
+                    } else {
+                        PathBuf::from(SPFS_DIR).join(extra_mount.dest.clone())
+                    };
+
+                    let mut cmd = tokio::process::Command::new(mount.clone());
+                    cmd.arg("--bind");
+                    cmd.arg(extra_mount.src.to_string_lossy().into_owned());
+                    cmd.arg(dest);
+                    tracing::debug!("About to run: {cmd:?}");
+
+                    match cmd.status().await {
+                        Err(err) => {
+                            return Err(Error::process_spawn_error("mount".to_owned(), err, None))
+                        }
+                        Ok(status) => match status.code() {
+                            Some(0) => (),
+                            _ => {
+                             return Err(format!(
+                                 "Failed to inject bind mount into the {SPFS_DIR} filesystem using: {cmd:?}"
+                             ).into())
+                            }
+                        },
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    async fn unmount_live_layers(&self, rt: &runtime::Runtime) -> Result<()> {
+        // Unmount the bind mounted items from the live layers
+        let live_layers = rt.live_layers();
+        if !live_layers.is_empty() {
+            tracing::debug!("unmounting the extra bind mounts from the {SPFS_DIR} filesystem ...");
+            let umount =
+                super::resolve::which("umount").unwrap_or_else(|| "/usr/bin/umount".into());
+
+            for layer in live_layers {
+                let injection_mounts = layer.bind_mounts();
+
+                for extra_mount in injection_mounts {
+                    let mut cmd = tokio::process::Command::new(umount.clone());
+                    cmd.arg(PathBuf::from(SPFS_DIR).join(extra_mount.dest.clone()));
+                    tracing::debug!("About to run: {cmd:?}");
+
+                    match cmd.status().await {
+                        Err(err) => {
+                            return Err(Error::process_spawn_error("umount".to_owned(), err, None))
+                        }
+                        Ok(status) => match status.code() {
+                            Some(0) => (),
+                            _ => return Err(format!("Failed to unmount a bind mount injected into the {SPFS_DIR} filesystem using: {cmd:?}").into()),
+                        },
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
     pub(crate) async fn mount_env_overlayfs<P: AsRef<Path>>(
         &self,
         rt: &runtime::Runtime,
@@ -437,7 +519,9 @@ where
                 Some(0) => Ok(()),
                 _ => Err("Failed to mount overlayfs".into()),
             },
-        }
+        }?;
+
+        self.mount_live_layers(rt).await
     }
 
     #[cfg(feature = "fuse-backend")]
@@ -447,7 +531,8 @@ where
 
     #[cfg(feature = "fuse-backend")]
     pub(crate) async fn mount_env_fuse(&self, rt: &runtime::Runtime) -> Result<()> {
-        self.mount_fuse_onto(rt, SPFS_DIR).await
+        self.mount_fuse_onto(rt, SPFS_DIR).await?;
+        self.mount_live_layers(rt).await
     }
 
     #[cfg(feature = "fuse-backend")]
@@ -763,7 +848,13 @@ where
     async fn unmount_env_fuse(&self, rt: &runtime::Runtime, lazy: bool) -> Result<()> {
         let mount_path = match rt.config.mount_backend {
             runtime::MountBackend::OverlayFsWithFuse => rt.config.lower_dir.as_path(),
-            runtime::MountBackend::FuseOnly => std::path::Path::new(SPFS_DIR),
+            runtime::MountBackend::FuseOnly => {
+                // Unmount any extra paths mounted in the depths of
+                // the fuse-only backend before fuse itself is
+                // unmounted to avoid issue with lazy unmounting.
+                self.unmount_live_layers(rt).await?;
+                std::path::Path::new(SPFS_DIR)
+            }
             runtime::MountBackend::OverlayFsWithRenders | runtime::MountBackend::WinFsp => {
                 return Ok(())
             }

crates/spfs/src/env_win.rs

@@ -2,6 +2,7 @@ use crate::tracking::EnvSpec;
 use crate::{runtime, Error, Result};
 
 pub const SPFS_DIR: &str = "C:\\spfs";
+pub const SPFS_DIR_PREFIX: &str = "C:\\spfs";
 
 /// Manages the configuration of an spfs runtime environment.
 ///

crates/spfs/src/error.rs

@@ -23,6 +23,8 @@ pub enum Error {
     #[error(transparent)]
     JSON(#[from] serde_json::Error),
     #[error(transparent)]
+    YAML(#[from] serde_yaml::Error),
+    #[error(transparent)]
     Config(#[from] config::ConfigError),
 
     #[error(transparent)]

crates/spfs/src/resolve.rs

@@ -141,10 +141,15 @@ pub async fn compute_environment_manifest(
 ) -> Result<tracking::Manifest> {
     let stack_futures: futures::stream::FuturesOrdered<_> = env
         .iter()
-        .map(|i| match i {
-            tracking::EnvSpecItem::Digest(d) => std::future::ready(Ok(*d)).boxed(),
-            tracking::EnvSpecItem::PartialDigest(p) => repo.resolve_full_digest(p).boxed(),
-            tracking::EnvSpecItem::TagSpec(t) => repo.resolve_tag(t).map_ok(|t| t.target).boxed(),
+        .filter_map(|i| match i {
+            tracking::EnvSpecItem::Digest(d) => Some(std::future::ready(Ok(*d)).boxed()),
+            tracking::EnvSpecItem::PartialDigest(p) => Some(repo.resolve_full_digest(p).boxed()),
+            tracking::EnvSpecItem::TagSpec(t) => {
+                Some(repo.resolve_tag(t).map_ok(|t| t.target).boxed())
+            }
+            // LiveLayers are stored in the runtime, not as spfs
+            // objects/layers, so this filters them out.
+            tracking::EnvSpecItem::LiveLayerFile(_) => None,
         })
         .collect();
     let stack: Vec<_> = stack_futures.try_collect().await?;

crates/spfs/src/runtime/mod.rs

@@ -21,8 +21,11 @@ pub use overlayfs::is_removed_entry;
 pub use storage::{
     makedirs_with_perms,
     Author,
+    BindMount,
     Config,
     Data,
+    LiveLayer,
+    LiveLayerFile,
     MountBackend,
     OwnedRuntime,
     Runtime,