diff --git a/Cargo.lock b/Cargo.lock index c8c32e392..dc3b8f293 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4174,6 +4174,7 @@ dependencies = [ "futures", "miette", "relative-path", + "serde_json", "spfs", "spk-schema", "spk-solve", @@ -4403,6 +4404,7 @@ dependencies = [ "console", "itertools 0.12.0", "miette", + "serde", "spfs", "spk-schema", "spk-storage", diff --git a/crates/spk-cli/common/src/env.rs b/crates/spk-cli/common/src/env.rs index cc43eae32..32759958d 100644 --- a/crates/spk-cli/common/src/env.rs +++ b/crates/spk-cli/common/src/env.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/imageworks/spk +use std::collections::HashMap; use std::ffi::{OsStr, OsString}; #[cfg(feature = "sentry")] use std::panic::catch_unwind; @@ -10,9 +11,16 @@ use std::sync::Arc; use miette::{Context, IntoDiagnostic, Result}; use once_cell::sync::Lazy; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, RequestedBy}; +use spk_schema::ident_ops::NormalizedTagStrategy; use spk_schema::{Package, VersionIdent}; use spk_solve::package_iterator::BUILD_SORT_TARGET; -use spk_solve::solution::{PackageSource, Solution}; +use spk_solve::solution::{ + PackageSolveData, + PackageSource, + PackagesToSolveData, + Solution, + SPK_SOLVE_EXTRA_DATA_KEY, +}; use spk_solve::validation::IMPOSSIBLE_CHECKS_TARGET; use spk_storage as storage; use tracing_subscriber::prelude::*; @@ -21,13 +29,13 @@ use crate::Error; /// Load the current environment from the spfs file system. pub async fn current_env() -> crate::Result { - match spfs::active_runtime().await { + let runtime = match spfs::active_runtime().await { Err(spfs::Error::NoActiveRuntime) => { return Err(Error::NoEnvironment); } Err(err) => return Err(err.into()), - Ok(_) => {} - } + Ok(rt) => rt, + }; let repo = Arc::new(storage::RepositoryHandle::Runtime(Default::default())); let mut packages_in_runtime_f = Vec::new(); @@ -82,18 +90,79 @@ pub async fn current_env() -> crate::Result { "return value from read_components_bulk expected to match input length" ); + // Pull the stored packages to solve data out of the runtime + let solve_data: PackagesToSolveData = + if let Some(json_data) = runtime.annotation(SPK_SOLVE_EXTRA_DATA_KEY).await? { + serde_json::from_str(&json_data).map_err(|err| Error::String(err.to_string()))? + } else { + // Fallback for older runtimes without any requested by extra data. + Default::default() + }; + + // For tracking source repos that we have seen before in the loop below. + let mut source_repos = HashMap::new(); + + // Used below when there is no entry for a resolved package in the + // requested_by_map. When used, it causes the first() requester + // below to fallback to RequestedBy::CurrentEnvironment + let no_package_solve_data: PackageSolveData = Default::default(); + for (spec, components) in packages_in_runtime .into_iter() .zip(components_in_runtime.into_iter()) { let range_ident = RangeIdent::equals(&spec.ident().to_any(), components.keys().cloned()); - let mut request = PkgRequest::new(range_ident, RequestedBy::CurrentEnvironment); + + let package_solve_data = if let Some(data) = solve_data.get(spec.ident()) { + data + } else { + &no_package_solve_data + }; + // The first requester is used in the PkgRequest constructor, + // and the rest are added to the new object after that. + let first_requester = match package_solve_data.requested_by.first() { + Some(r) => r.clone(), + None => RequestedBy::CurrentEnvironment, + }; + + let mut request = PkgRequest::new(range_ident, first_requester); + for requester in package_solve_data.requested_by.iter().skip(1) { + request.add_requester(requester.clone()); + } + request.prerelease_policy = Some(PreReleasePolicy::IncludeAll); - let repo = repo.clone(); + + let source_repo = match &package_solve_data.source_repo_name { + Some(name) => match source_repos.get(&name) { + Some(r) => Arc::clone(r), + None => { + // TODO: this code is repeated in few places in spk-cli + let r = match name.as_str() { + "local" => Arc::new(storage::local_repository().await?.into()), + name => Arc::new( + storage::remote_repository::<_, NormalizedTagStrategy>(name) + .await? + .into(), + ), + }; + // Store it, so we don't recreate it for any + // further resolved packages that came from the + // same named repo. + source_repos.insert(name, Arc::clone(&r)); + r + } + }, + // Falls back to using the current 'runtime' repo as the source repo. + None => repo.clone(), + }; + solution.add( request, spec, - PackageSource::Repository { repo, components }, + PackageSource::Repository { + repo: source_repo, + components, + }, ); } diff --git a/crates/spk-exec/Cargo.toml b/crates/spk-exec/Cargo.toml index 172cf023b..91338a15f 100644 --- a/crates/spk-exec/Cargo.toml +++ b/crates/spk-exec/Cargo.toml @@ -18,6 +18,7 @@ migration-to-components = [ async-stream = "0.3" futures = { workspace = true } relative-path = { workspace = true } +serde_json = { workspace = true } spfs = { workspace = true } spk-schema = { workspace = true } spk-storage = { workspace = true } diff --git a/crates/spk-exec/src/exec.rs b/crates/spk-exec/src/exec.rs index 1cb535088..47ecf330c 100644 --- a/crates/spk-exec/src/exec.rs +++ b/crates/spk-exec/src/exec.rs @@ -15,7 +15,7 @@ use spk_schema::foundation::format::{FormatIdent, FormatOptionMap}; use spk_schema::foundation::ident_component::Component; use spk_schema::prelude::*; use spk_schema::Spec; -use spk_solve::solution::{PackageSource, Solution}; +use spk_solve::solution::{PackageSource, Solution, SPK_SOLVE_EXTRA_DATA_KEY}; use spk_solve::RepositoryHandle; use spk_storage as storage; @@ -198,6 +198,20 @@ pub async fn setup_runtime(rt: &mut spfs::runtime::Runtime, solution: &Solution) let stack = resolve_runtime_layers(rt.config.mount_backend.requires_localization(), solution).await?; rt.status.stack = spfs::graph::Stack::from_iter(stack); + + // Store additional solve data all the resolved packages as extra + // data in the spfs runtime so future spk commands run inside the + // runtime can access it. + let solve_data = serde_json::to_string(&solution.packages_to_solve_data()) + .map_err(|err| Error::String(err.to_string()))?; + let spfs_config = spfs::Config::current()?; + rt.add_annotation( + SPK_SOLVE_EXTRA_DATA_KEY.to_string(), + solve_data, + spfs_config.filesystem.annotation_size_limit, + ) + .await?; + rt.save_state_to_storage().await?; spfs::remount_runtime(rt).await?; Ok(()) diff --git a/crates/spk-schema/crates/ident/src/request.rs b/crates/spk-schema/crates/ident/src/request.rs index 93c52adb7..3ac6990c2 100644 --- a/crates/spk-schema/crates/ident/src/request.rs +++ b/crates/spk-schema/crates/ident/src/request.rs @@ -571,7 +571,7 @@ where /// What made a PkgRequest, was it the command line, a test or a /// package build such as one resolved during a solve, or another /// package build resolved during a solve. -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub enum RequestedBy { /// From the command line CommandLine, @@ -638,7 +638,7 @@ impl std::fmt::Display for RequestedBy { } /// A desired package and set of restrictions on how it's selected. -#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct PkgRequest { pub pkg: RangeIdent, #[serde( diff --git a/crates/spk-solve/crates/solution/Cargo.toml b/crates/spk-solve/crates/solution/Cargo.toml index 1f94c024a..d58a5019c 100644 --- a/crates/spk-solve/crates/solution/Cargo.toml +++ b/crates/spk-solve/crates/solution/Cargo.toml @@ -17,6 +17,7 @@ migration-to-components = [ colored = { workspace = true } console = { workspace = true } itertools = { workspace = true } +serde = { workspace = true, features = ["derive"] } spfs = { workspace = true } spk-schema = { workspace = true } spk-storage = { workspace = true } diff --git a/crates/spk-solve/crates/solution/src/lib.rs b/crates/spk-solve/crates/solution/src/lib.rs index 33bddb985..24d3188e5 100644 --- a/crates/spk-solve/crates/solution/src/lib.rs +++ b/crates/spk-solve/crates/solution/src/lib.rs @@ -3,9 +3,11 @@ // https://github.com/imageworks/spk mod error; +mod package_solve_data; mod solution; pub use error::{Error, Result}; +pub use package_solve_data::{PackageSolveData, PackagesToSolveData, SPK_SOLVE_EXTRA_DATA_KEY}; pub use solution::{ find_highest_package_version, get_spfs_layers_to_packages, diff --git a/crates/spk-solve/crates/solution/src/package_solve_data.rs b/crates/spk-solve/crates/solution/src/package_solve_data.rs new file mode 100644 index 000000000..bcabc15e0 --- /dev/null +++ b/crates/spk-solve/crates/solution/src/package_solve_data.rs @@ -0,0 +1,61 @@ +// Copyright (c) Sony Pictures Imageworks, et al. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/imageworks/spk +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use spk_schema::ident::RequestedBy; +use spk_schema::BuildIdent; + +/// Key for extra data stored in spfs runtimes by spk when creating a +/// runtime and read back in by spk commands run inside that spfs/spk +/// environment. +pub const SPK_SOLVE_EXTRA_DATA_KEY: &str = "spk_solve"; + +/// Current data structure version number for PackageToSolveData +pub const PACKAGE_TO_SOLVE_DATA_VERSION: u32 = 1; + +/// Holds the extra solve related data for a package +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct PackageSolveData { + /// What a resolved package was requested by + pub requested_by: Vec, + /// Name of the repo the resolve package was found in. Optional + /// because embedded packages will have not have source repos. + pub source_repo_name: Option, +} + +/// The extra solve data for all the resolve packages for saving in +/// the spfs runtimes created by spk after a solver run +#[derive(Default, Serialize, Deserialize)] +pub struct PackagesToSolveData { + /// For tracking data structure changes + #[serde(deserialize_with = "ensure_version")] + version: u32, + /// Resolved package id to solve data mapping + data: BTreeMap, +} + +fn ensure_version<'de, D>(derserializer: D) -> std::result::Result +where + D: serde::Deserializer<'de>, +{ + let version = u32::deserialize(derserializer)?; + if version != PACKAGE_TO_SOLVE_DATA_VERSION { + return Err(serde::de::Error::custom(format!("PackagesToSolveData version mismatch. Required version {PACKAGE_TO_SOLVE_DATA_VERSION} but data is version {version}"))); + } + Ok(version) +} + +impl PackagesToSolveData { + pub fn get(&self, key: &BuildIdent) -> Option<&PackageSolveData> { + self.data.get(key) + } +} + +impl From> for PackagesToSolveData { + fn from(data: BTreeMap) -> Self { + let version = PACKAGE_TO_SOLVE_DATA_VERSION; + PackagesToSolveData { version, data } + } +} diff --git a/crates/spk-solve/crates/solution/src/solution.rs b/crates/spk-solve/crates/solution/src/solution.rs index 37a165835..514682f94 100644 --- a/crates/spk-solve/crates/solution/src/solution.rs +++ b/crates/spk-solve/crates/solution/src/solution.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/imageworks/spk -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt::Write; use std::iter::FromIterator; use std::sync::Arc; @@ -26,7 +26,7 @@ use spk_schema::version::Version; use spk_schema::{BuildEnv, BuildIdent, Package, Spec, SpecRecipe, VersionIdent}; use spk_storage::RepositoryHandle; -use crate::{Error, Result}; +use crate::{Error, PackageSolveData, PackagesToSolveData, Result}; const SOLUTION_FORMAT_EMPTY_REPORT: &str = "Nothing Installed"; const SOLUTION_FORMAT_HEADING: &str = "Installed Packages:\n"; @@ -556,6 +556,32 @@ impl Solution { let _ = write!(out, " {SOLUTION_FORMAT_FOOTER} {number_of_packages}"); out } + + /// Return a mapping of additional solve data for all the resolved + /// packages in the solution. + pub fn packages_to_solve_data(&self) -> PackagesToSolveData { + self.items() + .map(|sr| { + let source_repo_name = match &sr.source { + PackageSource::Repository { + repo, + components: _, + } => Some(repo.name().to_string()), + PackageSource::Embedded { .. } => None, + PackageSource::BuildFromSource { .. } => None, + PackageSource::SpkInternalTest => None, + }; + ( + sr.spec.ident().clone(), + PackageSolveData { + requested_by: sr.request.get_requesters(), + source_repo_name, + }, + ) + }) + .collect::>() + .into() + } } impl BuildEnv for Solution { diff --git a/cspell.json b/cspell.json index b89b1d431..8ab343f09 100644 --- a/cspell.json +++ b/cspell.json @@ -330,6 +330,7 @@ "depvalue", "depvar", "depver", + "derserializer", "devel", "devs", "devtoolset",