Merge pull request #938 from imageworks/flatbuffer-objects

Flatbuffers for Spfs Objects

.github/workflows/rust.yml

@@ -25,6 +25,11 @@ jobs:
         uses: crazy-max/ghaction-chocolatey@v3
         with:
           args: install protoc llvm winfsp make
+      - name: Install FlatBuffers Compiler
+        run: |
+          $url = "https://github.com/google/flatbuffers/releases/download/v23.5.26/Windows.flatc.binary.zip"
+          Invoke-WebRequest $url -OutFile C:\flatc.zip
+          Expand-Archive C:\flatc.zip -DestinationPath C:\Windows
       - name: Install Debug SPFS
         run: |
           make install-debug-spfs
@@ -55,6 +60,9 @@ jobs:
           apt-get install -y libcap2-bin sudo cmake tcsh rsync protobuf-compiler fuse libfuse-dev
           # spfs-fuse requires this option enabled
           echo user_allow_other >> /etc/fuse.conf
+          FB_REL=https://github.com/google/flatbuffers/releases/
+          curl --proto '=https' --tlsv1.2 -sSfL ${FB_REL}/download/v23.5.26/Linux.flatc.binary.g++-10.zip | funzip > /usr/bin/flatc
+          chmod +x /usr/bin/flatc
           rustup install nightly
           rustup component add clippy
           rustup component add rustfmt --toolchain nightly

Cargo.toml

@@ -5,8 +5,9 @@ codegen-units = 1
 [workspace]
 members = [
     "crates/progress_bar_derive_macro",
-    "crates/spfs-encoding",
     "crates/spfs",
+    "crates/spfs-encoding",
+    "crates/spfs-proto",
     "crates/spk-build",
     "crates/spk-cli/*",
     "crates/spfs-cli/*",
@@ -45,9 +46,11 @@ config = "0.14.0"
 console = "0.15"
 dashmap = "5.4.0"
 derive_builder = "0.20"
+data-encoding = "2.3"
 dirs = "5.0"
 dunce = "1.0.4"
 dyn-clone = "1.0"
+flatbuffers = "23.5.26"
 fuser = "0.14.0"
 futures = "0.3.28"
 futures-core = "0.3.28"

Makefile

@@ -46,9 +46,18 @@ clean: packages.clean
 
 .PHONY: lint
 lint: FEATURES?=server,spfs/server
-lint:
+lint: lint-fmt lint-clippy lint-docs
+
+.PHONY: lint-fmt
+lint-fmt:
 	$(CARGO) +nightly fmt --check
-	$(CARGO) clippy --tests $(cargo_features_arg) $(cargo_packages_arg) -- -Dwarnings
+
+.PHONY: lint-clippy
+lint-clippy:
+	$(CARGO) clippy --tests $(cargo_features_arg) $(cargo_packages_arg) $(CARGO_ARGS) -- -Dwarnings
+
+.PHONY: lint-docs
+lint-docs:
 	env RUSTDOCFLAGS="-Dwarnings" cargo doc --no-deps $(cargo_features_arg) $(cargo_packages_arg)
 
 .PHONY: format
@@ -73,7 +82,7 @@ release-spfs:
 .PHONY: test
 test: FEATURES?=server,spfs/server
 test:
-	spfs run - -- cargo test $(cargo_features_arg) $(cargo_packages_arg)
+	spfs run - -- cargo test $(cargo_features_arg) $(cargo_packages_arg) -- $(TEST_ARGS)
 
 .PHONY: converters
 converters:
@@ -92,7 +101,9 @@ spk-rpm:
 
 .PHONY: rpm-build
 rpm-build: rpm-buildenv
+	# ulimit for faster yum installs
 	docker build . \
+		--ulimit 'nofile=32768:32768' \
 		--target rpm_build \
 		--cache-from build_env \
 		-f rpmbuild.Dockerfile \
@@ -108,10 +119,10 @@ rpm-build: rpm-buildenv
 
 .PHONY: rpm-buildenv
 rpm-buildenv:
+	# ulimit for faster yum installs
 	docker build . \
+		--ulimit 'nofile=32768:32768' \
 		--target build_env \
 		--cache-from build_env \
 		-f rpmbuild.Dockerfile \
 		--tag build_env
-
-

crates/spfs-cli/cmd-enter/src/cmd_enter.rs

@@ -142,6 +142,8 @@ impl CmdEnter {
 
         let mut runtime = self.load_runtime(config).await?;
 
+        tracing::debug!("entering runtime: {runtime:#?}");
+
         if self.make_durable.enabled {
             if runtime.is_durable() {
                 return Err(spfs::Error::from("runtime is already durable").into());
@@ -241,6 +243,7 @@ impl CmdEnter {
         config: &spfs::Config,
     ) -> Result<Option<spfs::runtime::OwnedRuntime>> {
         let runtime = self.load_runtime(config).await?;
+        tracing::debug!("entering runtime: {runtime:#?}");
         if self.exit.enabled {
             todo!()
         } else if self.remount.enabled {

crates/spfs-cli/cmd-render/src/cmd_render.rs

@@ -156,7 +156,7 @@ impl CmdRender {
                 &render_summary_reporter as &dyn spfs::storage::fs::RenderReporter,
             ]),
         );
-        let stack = layers.into_iter().map(|l| l.manifest).collect();
+        let stack = layers.into_iter().map(|l| *l.manifest()).collect();
         renderer
             .render(&stack, self.strategy)
             .await

crates/spfs-cli/cmd-winfsp/src/cmd_winfsp.rs

@@ -75,7 +75,7 @@ impl cli::CommandName for CmdWinFsp {
 
 impl CmdWinFsp {
     fn run(&mut self, config: &spfs::Config) -> Result<i32> {
-        // the actual winfsp filesystem uses it's own threads, and
+        // the actual winfsp filesystem uses its own threads, and
         // the mount command only needs to send requests to the running
         // service, so a current thread runtime is appropriate
         let rt = tokio::runtime::Builder::new_current_thread()
@@ -215,7 +215,7 @@ impl CmdService {
 #[derive(Debug, Args)]
 struct CmdMount {
     /// The process id for which the mount will be visible, along
-    /// with all of it's children. Defaults to the calling process.
+    /// with all of its children. Defaults to the calling process.
     #[clap(long)]
     root_process: Option<u32>,
 

crates/spfs-cli/main/src/cmd_commit.rs

@@ -6,7 +6,7 @@ use std::path::PathBuf;
 
 use clap::Args;
 use miette::Result;
-use spfs::encoding::Encodable;
+use spfs::encoding::prelude::*;
 use spfs::prelude::*;
 
 /// Commit the current runtime state or a directory to storage
@@ -72,6 +72,10 @@ pub struct CmdCommit {
         required_unless_present = "path",
     )]
     kind: Option<String>,
+
+    /// Allow committing an empty layer or platform
+    #[clap(long, hide = true)]
+    allow_empty: bool,
 }
 
 impl CmdCommit {
@@ -82,7 +86,8 @@ impl CmdCommit {
             let committer = spfs::Committer::new(&repo)
                 .with_reporter(spfs::commit::ConsoleCommitReporter::default())
                 .with_max_concurrent_branches(self.max_concurrent_branches)
-                .with_max_concurrent_blobs(self.max_concurrent_blobs);
+                .with_max_concurrent_blobs(self.max_concurrent_blobs)
+                .with_allow_empty(self.allow_empty);
             if self.hash_while_committing {
                 let committer = committer
                     .with_blob_hasher(spfs::commit::WriteToRepositoryBlobHasher { repo: &repo });
@@ -92,7 +97,7 @@ impl CmdCommit {
             }
         };
 
-        tracing::info!(digest = ?result.digest()?, "created");
+        tracing::info!(digest=%result.digest()?, "created");
         for tag in self.tags.iter() {
             let tag_spec = match spfs::tracking::TagSpec::parse(tag) {
                 Ok(tag_spec) => tag_spec,
@@ -122,11 +127,11 @@ impl CmdCommit {
     {
         if let Some(path) = &self.path {
             let manifest = committer.commit_dir(path).await?;
-            if manifest.is_empty() {
+            if manifest.is_empty() && !self.allow_empty {
                 return Err(spfs::Error::NothingToCommit);
             }
             return Ok(repo
-                .create_layer(&spfs::graph::Manifest::from(&manifest))
+                .create_layer(&manifest.to_graph_manifest())
                 .await?
                 .into());
         }

crates/spfs-cli/main/src/cmd_info.rs

@@ -9,7 +9,6 @@ use colored::*;
 use miette::Result;
 use spfs::env::SPFS_DIR;
 use spfs::find_path::ObjectPathEntry;
-use spfs::graph::Object;
 use spfs::io::{self, DigestFormat, Pluralize};
 use spfs::prelude::*;
 use spfs::{self};
@@ -104,8 +103,9 @@ impl CmdInfo {
         repo: &spfs::storage::RepositoryHandle,
         verbosity: usize,
     ) -> Result<()> {
-        match obj {
-            Object::Platform(obj) => {
+        use spfs::graph::object::Enum;
+        match obj.into_enum() {
+            Enum::Platform(obj) => {
                 println!(
                     "{}:\n{}",
                     self.format_digest(obj.digest()?, repo).await?,
@@ -117,15 +117,15 @@ impl CmdInfo {
                     self.format_digest(obj.digest()?, repo).await?
                 );
                 println!("{}:", "stack (top-down)".bright_blue());
-                for reference in obj.stack.to_top_down() {
+                for reference in obj.to_stack().to_top_down() {
                     println!("  - {}", self.format_digest(reference, repo).await?);
                     if self.follow {
                         self.to_process.push_back(reference.to_string());
                     }
                 }
             }
 
-            Object::Layer(obj) => {
+            Enum::Layer(obj) => {
                 println!(
                     "{}:\n{}",
                     self.format_digest(obj.digest()?, repo).await?,
@@ -139,14 +139,14 @@ impl CmdInfo {
                 println!(
                     " {} {}",
                     "manifest:".bright_blue(),
-                    self.format_digest(obj.manifest, repo).await?
+                    self.format_digest(*obj.manifest(), repo).await?
                 );
                 if self.follow {
-                    self.to_process.push_back(obj.manifest.to_string());
+                    self.to_process.push_back(obj.manifest().to_string());
                 }
             }
 
-            Object::Manifest(obj) => {
+            Enum::Manifest(obj) => {
                 println!(
                     "{}:\n{}",
                     self.format_digest(obj.digest()?, repo).await?,
@@ -174,24 +174,23 @@ impl CmdInfo {
                 }
             }
 
-            Object::Blob(obj) => {
+            Enum::Blob(obj) => {
                 println!(
                     "{}:\n {}:",
-                    self.format_digest(obj.payload, repo).await?,
+                    self.format_digest(*obj.payload(), repo).await?,
                     "blob".green()
                 );
                 println!(
                     " {} {}",
                     "digest:".bright_blue(),
-                    self.format_digest(obj.payload, repo).await?
+                    self.format_digest(*obj.payload(), repo).await?
                 );
                 println!(
                     " {} {}",
                     "size:".bright_blue(),
-                    spfs::io::format_size(obj.size)
+                    spfs::io::format_size(obj.size())
                 );
             }
-            Object::Tree(_) | Object::Mask => println!("{obj:?}"),
         }
         Ok(())
     }

crates/spfs-cli/main/src/cmd_read.rs

@@ -32,9 +32,9 @@ impl CmdRead {
         tracing::info!(target: "sentry", "using repo: {}", repo.address());
 
         let item = repo.read_ref(&self.reference.to_string()).await?;
-        use spfs::graph::Object;
-        let blob = match item {
-            Object::Blob(blob) => blob,
+        use spfs::graph::object::Enum;
+        let blob = match item.to_enum() {
+            Enum::Blob(blob) => blob,
             _ => {
                 let path = match &self.path {
                     None => {
@@ -58,7 +58,7 @@ impl CmdRead {
             }
         };
 
-        let (mut payload, filename) = repo.open_payload(blob.digest()).await?;
+        let (mut payload, filename) = repo.open_payload(*blob.digest()).await?;
         tokio::io::copy(&mut payload, &mut tokio::io::stdout())
             .await
             .map_err(|err| Error::StorageReadError("copy of payload to stdout", filename, err))?;