Stores/restores the solution's 'requested by' data into/from the spfs…

… runtime

Signed-off-by: David Gilligan-Cook <dcook@imageworks.com>

crates/spk-cli/common/src/env.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // https://github.com/imageworks/spk
 
+use std::collections::HashMap;
 use std::ffi::{OsStr, OsString};
 #[cfg(feature = "sentry")]
 use std::panic::catch_unwind;
@@ -10,9 +11,16 @@ use std::sync::Arc;
 use miette::{Context, IntoDiagnostic, Result};
 use once_cell::sync::Lazy;
 use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, RequestedBy};
+use spk_schema::ident_ops::NormalizedTagStrategy;
 use spk_schema::{Package, VersionIdent};
 use spk_solve::package_iterator::BUILD_SORT_TARGET;
-use spk_solve::solution::{PackageSource, Solution};
+use spk_solve::solution::{
+    PackageSolveData,
+    PackageSource,
+    PackagesToSolveData,
+    Solution,
+    SPK_SOLVE_EXTRA_DATA_KEY,
+};
 use spk_solve::validation::IMPOSSIBLE_CHECKS_TARGET;
 use spk_storage as storage;
 use tracing_subscriber::prelude::*;
@@ -21,13 +29,13 @@ use crate::Error;
 
 /// Load the current environment from the spfs file system.
 pub async fn current_env() -> crate::Result<Solution> {
-    match spfs::active_runtime().await {
+    let runtime = match spfs::active_runtime().await {
         Err(spfs::Error::NoActiveRuntime) => {
             return Err(Error::NoEnvironment);
         }
         Err(err) => return Err(err.into()),
-        Ok(_) => {}
-    }
+        Ok(rt) => rt,
+    };
 
     let repo = Arc::new(storage::RepositoryHandle::Runtime(Default::default()));
     let mut packages_in_runtime_f = Vec::new();
@@ -82,18 +90,79 @@ pub async fn current_env() -> crate::Result<Solution> {
         "return value from read_components_bulk expected to match input length"
     );
 
+    // Pull the stored packages to solve data out of the runtime
+    let solve_data: PackagesToSolveData =
+        if let Some(json_data) = runtime.annotation(SPK_SOLVE_EXTRA_DATA_KEY).await? {
+            serde_json::from_str(&json_data).map_err(|err| Error::String(err.to_string()))?
+        } else {
+            // Fallback for older runtimes without any requested by extra data.
+            Default::default()
+        };
+
+    // For tracking source repos that we have seen before in the loop below.
+    let mut source_repos = HashMap::new();
+
+    // Used below when there is no entry for a resolved package in the
+    // requested_by_map. When used, it causes the first() requester
+    // below to fallback to RequestedBy::CurrentEnvironment
+    let no_package_solve_data: PackageSolveData = Default::default();
+
     for (spec, components) in packages_in_runtime
         .into_iter()
         .zip(components_in_runtime.into_iter())
     {
         let range_ident = RangeIdent::equals(&spec.ident().to_any(), components.keys().cloned());
-        let mut request = PkgRequest::new(range_ident, RequestedBy::CurrentEnvironment);
+
+        let package_solve_data = if let Some(data) = solve_data.get(spec.ident()) {
+            data
+        } else {
+            &no_package_solve_data
+        };
+        // The first requester is used in the PkgRequest constructor,
+        // and the rest are added to the new object after that.
+        let first_requester = match package_solve_data.requested_by.first() {
+            Some(r) => r.clone(),
+            None => RequestedBy::CurrentEnvironment,
+        };
+
+        let mut request = PkgRequest::new(range_ident, first_requester);
+        for requester in package_solve_data.requested_by.iter().skip(1) {
+            request.add_requester(requester.clone());
+        }
+
         request.prerelease_policy = Some(PreReleasePolicy::IncludeAll);
-        let repo = repo.clone();
+
+        let source_repo = match &package_solve_data.source_repo_name {
+            Some(name) => match source_repos.get(&name) {
+                Some(r) => Arc::clone(r),
+                None => {
+                    // TODO: this code is repeated in few places in spk-cli
+                    let r = match name.as_str() {
+                        "local" => Arc::new(storage::local_repository().await?.into()),
+                        name => Arc::new(
+                            storage::remote_repository::<_, NormalizedTagStrategy>(name)
+                                .await?
+                                .into(),
+                        ),
+                    };
+                    // Store it, so we don't recreate it for any
+                    // further resolved packages that came from the
+                    // same named repo.
+                    source_repos.insert(name, Arc::clone(&r));
+                    r
+                }
+            },
+            // Falls back to using the current 'runtime' repo as the source repo.
+            None => repo.clone(),
+        };
+
         solution.add(
             request,
             spec,
-            PackageSource::Repository { repo, components },
+            PackageSource::Repository {
+                repo: source_repo,
+                components,
+            },
         );
     }
 

crates/spk-exec/Cargo.toml

@@ -18,6 +18,7 @@ migration-to-components = [
 async-stream = "0.3"
 futures = { workspace = true }
 relative-path = { workspace = true }
+serde_json = { workspace = true }
 spfs = { workspace = true }
 spk-schema = { workspace = true }
 spk-storage = { workspace = true }

crates/spk-exec/src/exec.rs

@@ -15,7 +15,7 @@ use spk_schema::foundation::format::{FormatIdent, FormatOptionMap};
 use spk_schema::foundation::ident_component::Component;
 use spk_schema::prelude::*;
 use spk_schema::Spec;
-use spk_solve::solution::{PackageSource, Solution};
+use spk_solve::solution::{PackageSource, Solution, SPK_SOLVE_EXTRA_DATA_KEY};
 use spk_solve::RepositoryHandle;
 use spk_storage as storage;
 
@@ -198,6 +198,20 @@ pub async fn setup_runtime(rt: &mut spfs::runtime::Runtime, solution: &Solution)
     let stack =
         resolve_runtime_layers(rt.config.mount_backend.requires_localization(), solution).await?;
     rt.status.stack = spfs::graph::Stack::from_iter(stack);
+
+    // Store additional solve data all the resolved packages as extra
+    // data in the spfs runtime so future spk commands run inside the
+    // runtime can access it.
+    let solve_data = serde_json::to_string(&solution.packages_to_solve_data())
+        .map_err(|err| Error::String(err.to_string()))?;
+    let spfs_config = spfs::Config::current()?;
+    rt.add_annotation(
+        SPK_SOLVE_EXTRA_DATA_KEY.to_string(),
+        solve_data,
+        spfs_config.filesystem.annotation_size_limit,
+    )
+    .await?;
+
     rt.save_state_to_storage().await?;
     spfs::remount_runtime(rt).await?;
     Ok(())

crates/spk-schema/crates/ident/src/request.rs

@@ -571,7 +571,7 @@ where
 /// What made a PkgRequest, was it the command line, a test or a
 /// package build such as one resolved during a solve, or another
 /// package build resolved during a solve.
-#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
+#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
 pub enum RequestedBy {
     /// From the command line
     CommandLine,
@@ -638,7 +638,7 @@ impl std::fmt::Display for RequestedBy {
 }
 
 /// A desired package and set of restrictions on how it's selected.
-#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)]
+#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
 pub struct PkgRequest {
     pub pkg: RangeIdent,
     #[serde(

crates/spk-solve/crates/solution/Cargo.toml

@@ -17,6 +17,7 @@ migration-to-components = [
 colored = { workspace = true }
 console = { workspace = true }
 itertools = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
 spfs = { workspace = true }
 spk-schema = { workspace = true }
 spk-storage = { workspace = true }

crates/spk-solve/crates/solution/src/lib.rs

@@ -3,9 +3,11 @@
 // https://github.com/imageworks/spk
 
 mod error;
+mod package_solve_data;
 mod solution;
 
 pub use error::{Error, Result};
+pub use package_solve_data::{PackageSolveData, PackagesToSolveData, SPK_SOLVE_EXTRA_DATA_KEY};
 pub use solution::{
     find_highest_package_version,
     get_spfs_layers_to_packages,

crates/spk-solve/crates/solution/src/package_solve_data.rs

@@ -0,0 +1,49 @@
+// Copyright (c) Sony Pictures Imageworks, et al.
+// SPDX-License-Identifier: Apache-2.0
+// https://github.com/imageworks/spk
+use std::collections::BTreeMap;
+
+use serde::{Deserialize, Serialize};
+use spk_schema::ident::RequestedBy;
+use spk_schema::BuildIdent;
+
+/// Key for extra data stored in spfs runtimes by spk when creating a
+/// runtime and read back in by spk commands run inside that spfs/spk
+/// environment.
+pub const SPK_SOLVE_EXTRA_DATA_KEY: &str = "SPK_solve";
+
+/// Current data structure version number for PackageToSolveData
+pub const PACKAGE_TO_SOLVE_DATA_VERSION: u32 = 1;
+
+/// Holds the extra solve related data for a package
+#[derive(Clone, Debug, Serialize, Deserialize, Default)]
+pub struct PackageSolveData {
+    /// What a resolved package was requested by
+    pub requested_by: Vec<RequestedBy>,
+    /// Name of the repo the resolve package was found in. Optional
+    /// because embedded packages will have not have source repos.
+    pub source_repo_name: Option<String>,
+}
+
+/// The extra solve data for all the resolve packages for saving in
+/// the spfs runtime's created by spk after a solver run
+#[derive(Default, Serialize, Deserialize)]
+pub struct PackagesToSolveData {
+    /// For tracking data structure changes
+    version: u32,
+    /// Resolved package id to solve data mapping
+    data: BTreeMap<BuildIdent, PackageSolveData>,
+}
+
+impl PackagesToSolveData {
+    pub fn get(&self, key: &BuildIdent) -> Option<&PackageSolveData> {
+        self.data.get(key)
+    }
+}
+
+impl From<BTreeMap<BuildIdent, PackageSolveData>> for PackagesToSolveData {
+    fn from(data: BTreeMap<BuildIdent, PackageSolveData>) -> Self {
+        let version = PACKAGE_TO_SOLVE_DATA_VERSION;
+        PackagesToSolveData { version, data }
+    }
+}

crates/spk-solve/crates/solution/src/solution.rs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // https://github.com/imageworks/spk
 
-use std::collections::{BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::fmt::Write;
 use std::iter::FromIterator;
 use std::sync::Arc;
@@ -26,7 +26,7 @@ use spk_schema::version::Version;
 use spk_schema::{BuildEnv, BuildIdent, Package, Spec, SpecRecipe, VersionIdent};
 use spk_storage::RepositoryHandle;
 
-use crate::{Error, Result};
+use crate::{Error, PackageSolveData, PackagesToSolveData, Result};
 
 const SOLUTION_FORMAT_EMPTY_REPORT: &str = "Nothing Installed";
 const SOLUTION_FORMAT_HEADING: &str = "Installed Packages:\n";
@@ -556,6 +556,32 @@ impl Solution {
         let _ = write!(out, " {SOLUTION_FORMAT_FOOTER} {number_of_packages}");
         out
     }
+
+    /// Return a mapping of additional solve data for all the resolved
+    /// packages in the solution.
+    pub fn packages_to_solve_data(&self) -> PackagesToSolveData {
+        self.items()
+            .map(|sr| {
+                let source_repo_name = match &sr.source {
+                    PackageSource::Repository {
+                        repo,
+                        components: _,
+                    } => Some(repo.name().to_string()),
+                    PackageSource::Embedded { .. } => None,
+                    PackageSource::BuildFromSource { .. } => None,
+                    PackageSource::SpkInternalTest => None,
+                };
+                (
+                    sr.spec.ident().clone(),
+                    PackageSolveData {
+                        requested_by: sr.request.get_requesters(),
+                        source_repo_name,
+                    },
+                )
+            })
+            .collect::<BTreeMap<BuildIdent, PackageSolveData>>()
+            .into()
+    }
 }
 
 impl BuildEnv for Solution {