diff --git a/Cargo.lock b/Cargo.lock index 9b2b75ae8c..29e6ed48df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3999,6 +3999,7 @@ dependencies = [ "async-stream", "futures", "relative-path", + "serde_json", "spfs", "spk-schema", "spk-solve", @@ -4240,6 +4241,7 @@ dependencies = [ "colored", "console", "itertools 0.10.5", + "serde", "spfs", "spk-schema", "spk-storage", diff --git a/crates/spfs-cli/main/src/cmd_run.rs b/crates/spfs-cli/main/src/cmd_run.rs index 34aea1063c..39b6296af2 100644 --- a/crates/spfs-cli/main/src/cmd_run.rs +++ b/crates/spfs-cli/main/src/cmd_run.rs @@ -249,9 +249,7 @@ impl CmdRun { if !data.is_empty() { tracing::debug!("with extra external data: {data:?}"); for (key, value) in data { - runtime - .add_external_data(key, value, config.filesystem.external_data_size_limit) - .await?; + runtime.add_external_data(key, value).await?; } } diff --git a/crates/spfs/src/runtime/storage.rs b/crates/spfs/src/runtime/storage.rs index 82139198ea..c4a847d3a5 100644 --- a/crates/spfs/src/runtime/storage.rs +++ b/crates/spfs/src/runtime/storage.rs @@ -702,8 +702,11 @@ impl Runtime { &mut self, key: String, value: String, - size_limit: usize, + //size_limit: usize, ) -> Result<()> { + let config = crate::config::get_config()?; + let size_limit = config.filesystem.external_data_size_limit; + tracing::debug!( "about to insert external data key: {} => value: {} [len: {} > {}]", key, diff --git a/crates/spfs/src/runtime/storage_test.rs b/crates/spfs/src/runtime/storage_test.rs index b638f1f887..6ea4d2103f 100644 --- a/crates/spfs/src/runtime/storage_test.rs +++ b/crates/spfs/src/runtime/storage_test.rs @@ -152,7 +152,6 @@ async fn test_storage_runtime_with_external_data(tmpdir: tempfile::TempDir) { .unwrap(), ); let storage = Storage::new(repo); - let limit: usize = 16 * 1024; let keep_runtime = false; let extra_mounts = None; @@ -164,15 +163,12 @@ async fn test_storage_runtime_with_external_data(tmpdir: tempfile::TempDir) { // Test - insert new data let key = "somefield".to_string(); let value = "somevalue".to_string(); - assert!(runtime - .add_external_data(key.clone(), value, limit) - .await - .is_ok()); + assert!(runtime.add_external_data(key.clone(), value).await.is_ok()); // Test - insert over existing data let value2 = "someothervalue".to_string(); assert!(runtime - .add_external_data(key.clone(), value2.clone(), limit) + .add_external_data(key.clone(), value2.clone()) .await .is_ok()); @@ -240,7 +236,6 @@ async fn test_storage_runtime_with_external_data_all(tmpdir: tempfile::TempDir) .unwrap(), ); let storage = Storage::new(repo); - let limit: usize = 16 * 1024; let keep_runtime = false; let extra_mounts = None; @@ -253,14 +248,14 @@ async fn test_storage_runtime_with_external_data_all(tmpdir: tempfile::TempDir) let key = "somefield".to_string(); let value = "somevalue".to_string(); assert!(runtime - .add_external_data(key.clone(), value.clone(), limit) + .add_external_data(key.clone(), value.clone()) .await .is_ok()); let key2 = "somefield2".to_string(); let value2 = "somevalue2".to_string(); assert!(runtime - .add_external_data(key2.clone(), value2.clone(), limit) + .add_external_data(key2.clone(), value2.clone()) .await .is_ok()); diff --git a/crates/spk-cli/common/src/env.rs b/crates/spk-cli/common/src/env.rs index 7a0104dfd1..8aac0abdcd 100644 --- a/crates/spk-cli/common/src/env.rs +++ b/crates/spk-cli/common/src/env.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/imageworks/spk +use std::collections::HashMap; use std::ffi::{OsStr, OsString}; #[cfg(feature = "sentry")] use std::panic::catch_unwind; @@ -12,7 +13,13 @@ use once_cell::sync::Lazy; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, RequestedBy}; use spk_schema::{Package, VersionIdent}; use spk_solve::package_iterator::BUILD_SORT_TARGET; -use spk_solve::solution::{PackageSource, Solution}; +use spk_solve::solution::{ + PackageSolveData, + PackageSource, + PackagesToSolveData, + Solution, + SPK_SOLVE_EXTRA_DATA_KEY, +}; use spk_solve::validation::IMPOSSIBLE_CHECKS_TARGET; use spk_storage as storage; use tracing_subscriber::prelude::*; @@ -21,13 +28,13 @@ use crate::Error; /// Load the current environment from the spfs file system. pub async fn current_env() -> crate::Result { - match spfs::active_runtime().await { + let runtime = match spfs::active_runtime().await { Err(spfs::Error::NoActiveRuntime) => { return Err(Error::NoEnvironment); } Err(err) => return Err(err.into()), - Ok(_) => {} - } + Ok(rt) => rt, + }; let repo = Arc::new(storage::RepositoryHandle::Runtime(Default::default())); let mut packages_in_runtime_f = Vec::new(); @@ -82,18 +89,75 @@ pub async fn current_env() -> crate::Result { "return value from read_components_bulk expected to match input length" ); + // Pull the stored packages to solve data out of the runtime + let solve_data: PackagesToSolveData = + if let Some(json_data) = runtime.external_data(SPK_SOLVE_EXTRA_DATA_KEY).await? { + serde_json::from_str(&json_data).map_err(|err| Error::String(err.to_string()))? + } else { + // Fallback for older runtimes without any requested by extra data. + Default::default() + }; + + // For tracking source repos that we have seen before in the loop below. + let mut source_repos = HashMap::new(); + + // Used below when there is no entry for a resolved package in the + // requested_by_map. When used, it causes the first() requester + // below to fallback to RequestedBy::CurrentEnvironment + let no_package_solve_data: PackageSolveData = Default::default(); + for (spec, components) in packages_in_runtime .into_iter() .zip(components_in_runtime.into_iter()) { let range_ident = RangeIdent::equals(&spec.ident().to_any(), components.keys().cloned()); - let mut request = PkgRequest::new(range_ident, RequestedBy::CurrentEnvironment); + + let package_solve_data = if let Some(data) = solve_data.get(spec.ident()) { + data + } else { + &no_package_solve_data + }; + // The first requester is used in the PkgRequest constructor, + // and the rest are added to the new object after that. + let first_requester = match package_solve_data.requested_by.first() { + Some(r) => r.clone(), + None => RequestedBy::CurrentEnvironment, + }; + + let mut request = PkgRequest::new(range_ident, first_requester); + for requester in package_solve_data.requested_by.iter().skip(1) { + request.add_requester(requester.clone()); + } + request.prerelease_policy = Some(PreReleasePolicy::IncludeAll); - let repo = repo.clone(); + + let source_repo = match &package_solve_data.source_repo_name { + Some(name) => match source_repos.get(&name) { + Some(r) => Arc::clone(r), + None => { + // TODO: this code is repeated in few places in spk-cli + let r = match name.as_str() { + "local" => Arc::new(storage::local_repository().await?.into()), + name => Arc::new(storage::remote_repository(name).await?.into()), + }; + // Store it, so we don't recreate it for any + // further resolved packages that came from the + // same named repo. + source_repos.insert(name, Arc::clone(&r)); + r + } + }, + // Falls back to using the current 'runtime' repo as the source repo. + None => repo.clone(), + }; + solution.add( request, spec, - PackageSource::Repository { repo, components }, + PackageSource::Repository { + repo: source_repo, + components, + }, ); } diff --git a/crates/spk-exec/Cargo.toml b/crates/spk-exec/Cargo.toml index 6d4fe4ea73..809ca54dfe 100644 --- a/crates/spk-exec/Cargo.toml +++ b/crates/spk-exec/Cargo.toml @@ -15,6 +15,7 @@ migration-to-components = [ async-stream = "0.3" futures = { workspace = true } relative-path = "1.3" +serde_json = { workspace = true } spfs = { path = "../spfs" } spk-schema = { path = '../spk-schema' } spk-storage = { path = "../spk-storage" } diff --git a/crates/spk-exec/src/exec.rs b/crates/spk-exec/src/exec.rs index 374253016a..c8c758635d 100644 --- a/crates/spk-exec/src/exec.rs +++ b/crates/spk-exec/src/exec.rs @@ -16,7 +16,7 @@ use spk_schema::foundation::format::{FormatIdent, FormatOptionMap}; use spk_schema::foundation::ident_component::Component; use spk_schema::prelude::*; use spk_schema::Spec; -use spk_solve::solution::{PackageSource, Solution}; +use spk_solve::solution::{PackageSource, Solution, SPK_SOLVE_EXTRA_DATA_KEY}; use spk_solve::RepositoryHandle; use spk_storage as storage; @@ -194,6 +194,15 @@ pub async fn setup_runtime(rt: &mut spfs::runtime::Runtime, solution: &Solution) let stack = resolve_runtime_layers(rt.config.mount_backend.requires_localization(), solution).await?; rt.status.stack = stack; + + // Store additional solve data all the resolved packages as extra + // data in the spfs runtime so future spk commands run inside the + // runtime can access it. + let solve_data = serde_json::to_string(&solution.packages_to_solve_data()) + .map_err(|err| Error::String(err.to_string()))?; + rt.add_external_data(SPK_SOLVE_EXTRA_DATA_KEY.to_string(), solve_data) + .await?; + rt.save_state_to_storage().await?; spfs::remount_runtime(rt).await?; Ok(()) diff --git a/crates/spk-schema/crates/ident/src/request.rs b/crates/spk-schema/crates/ident/src/request.rs index e4fe1798e2..18a69fdc37 100644 --- a/crates/spk-schema/crates/ident/src/request.rs +++ b/crates/spk-schema/crates/ident/src/request.rs @@ -539,7 +539,7 @@ where /// What made a PkgRequest, was it the command line, a test or a /// package build such as one resolved during a solve, or another /// package build resolved during a solve. -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub enum RequestedBy { /// From the command line CommandLine, @@ -606,7 +606,7 @@ impl std::fmt::Display for RequestedBy { } /// A desired package and set of restrictions on how it's selected. -#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct PkgRequest { pub pkg: RangeIdent, #[serde( diff --git a/crates/spk-solve/crates/solution/Cargo.toml b/crates/spk-solve/crates/solution/Cargo.toml index 74c5d03386..cb5929fd5f 100644 --- a/crates/spk-solve/crates/solution/Cargo.toml +++ b/crates/spk-solve/crates/solution/Cargo.toml @@ -14,7 +14,8 @@ migration-to-components = [ colored = { workspace = true } console = { workspace = true } itertools = "0.10" -spfs = { version = '0.34.6', path = "../../../spfs" } +serde = { workspace = true, features = ["derive"] } +spfs = { path = "../../../spfs" } spk-schema = { path = "../../../spk-schema" } spk-storage = { path = "../../../spk-storage" } thiserror = { workspace = true } diff --git a/crates/spk-solve/crates/solution/src/lib.rs b/crates/spk-solve/crates/solution/src/lib.rs index f5e47de10c..08ccb4c9e4 100644 --- a/crates/spk-solve/crates/solution/src/lib.rs +++ b/crates/spk-solve/crates/solution/src/lib.rs @@ -6,9 +6,11 @@ #![warn(clippy::fn_params_excessive_bools)] mod error; +mod package_solve_data; mod solution; pub use error::{Error, Result}; +pub use package_solve_data::{PackageSolveData, PackagesToSolveData, SPK_SOLVE_EXTRA_DATA_KEY}; pub use solution::{ find_highest_package_version, get_spfs_layers_to_packages, diff --git a/crates/spk-solve/crates/solution/src/package_solve_data.rs b/crates/spk-solve/crates/solution/src/package_solve_data.rs new file mode 100644 index 0000000000..0421fa6f4c --- /dev/null +++ b/crates/spk-solve/crates/solution/src/package_solve_data.rs @@ -0,0 +1,49 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use spk_schema::ident::RequestedBy; +use spk_schema::BuildIdent; + +/// Key for extra data stored in spfs runtimes by spk when creating a +/// runtime and read back in by spk commands run inside that spfs/spk +/// environment. +pub const SPK_SOLVE_EXTRA_DATA_KEY: &str = "SPK_solve"; + +/// Current data structure version number for PackageToSolveData +pub const PACKAGE_TO_SOLVE_DATA_VERSION: u32 = 1; + +/// Holds the extra solve related data for a package +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct PackageSolveData { + /// What a resolved package was requested by + pub requested_by: Vec, + /// Name of the repo the resolve package was found in. Optional + /// because embedded packages will have not have source repos. + pub source_repo_name: Option, +} + +/// The extra solve data for all the resolve packages for saving in +/// the spfs runtime's created by spk after a solver run +#[derive(Default, Serialize, Deserialize)] +pub struct PackagesToSolveData { + /// For tracking data structure changes + version: u32, + /// Resolved package id to solve data mapping + data: BTreeMap, +} + +impl PackagesToSolveData { + pub fn get(&self, key: &BuildIdent) -> Option<&PackageSolveData> { + self.data.get(key) + } +} + +impl From> for PackagesToSolveData { + fn from(data: BTreeMap) -> Self { + let version = PACKAGE_TO_SOLVE_DATA_VERSION; + PackagesToSolveData { version, data } + } +} diff --git a/crates/spk-solve/crates/solution/src/solution.rs b/crates/spk-solve/crates/solution/src/solution.rs index b495382c6c..824193ce28 100644 --- a/crates/spk-solve/crates/solution/src/solution.rs +++ b/crates/spk-solve/crates/solution/src/solution.rs @@ -1,7 +1,7 @@ // Copyright (c) Sony Pictures Imageworks, et al. // SPDX-License-Identifier: Apache-2.0 // https://github.com/imageworks/spk -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt::Write; use std::iter::FromIterator; use std::sync::Arc; @@ -25,7 +25,7 @@ use spk_schema::version::Version; use spk_schema::{BuildEnv, BuildIdent, Package, Spec, SpecRecipe, VersionIdent}; use spk_storage::RepositoryHandle; -use crate::{Error, Result}; +use crate::{Error, PackageSolveData, PackagesToSolveData, Result}; const SOLUTION_FORMAT_EMPTY_REPORT: &str = "Nothing Installed"; const SOLUTION_FORMAT_HEADING: &str = "Installed Packages:\n"; @@ -543,6 +543,32 @@ impl Solution { let _ = write!(out, " {SOLUTION_FORMAT_FOOTER} {number_of_packages}"); out } + + /// Return a mapping of additional solve data for all the resolved + /// packages in the solution. + pub fn packages_to_solve_data(&self) -> PackagesToSolveData { + self.items() + .map(|sr| { + let source_repo_name = match &sr.source { + PackageSource::Repository { + repo, + components: _, + } => Some(repo.name().to_string()), + PackageSource::Embedded { .. } => None, + PackageSource::BuildFromSource { .. } => None, + PackageSource::SpkInternalTest => None, + }; + ( + sr.spec.ident().clone(), + PackageSolveData { + requested_by: sr.request.get_requesters(), + source_repo_name, + }, + ) + }) + .collect::>() + .into() + } } impl BuildEnv for Solution {