Add --allow-with-existing-label to publish (#992)

Adds metadata() method to recipe and its implementations to get at the metadata.

---------

Signed-off-by: David Gilligan-Cook <dcook@imageworks.com>

crates/spk-cli/common/src/lib.rs

@@ -20,7 +20,7 @@ pub use env::{configure_logging, current_env, spk_exe};
 pub use error::{Error, Result, TestError};
 pub use exec::build_required_packages;
 use once_cell::sync::Lazy;
-pub use publish::Publisher;
+pub use publish::{PublishLabel, Publisher};
 pub use with_version_and_build_set::{DefaultBuildStrategy, DefaultVersionStrategy};
 
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");

crates/spk-cli/common/src/publish.rs

@@ -2,11 +2,12 @@
 // SPDX-License-Identifier: Apache-2.0
 // https://github.com/imageworks/spk
 
+use std::str::FromStr;
 use std::sync::Arc;
 
 use spk_schema::foundation::format::{FormatComponents, FormatIdent};
 use spk_schema::foundation::ident_component::ComponentSet;
-use spk_schema::{AnyIdent, BuildIdent, Package, Recipe};
+use spk_schema::{AnyIdent, BuildIdent, Package, Recipe, VersionIdent};
 use spk_storage as storage;
 use storage::{with_cache_policy, CachePolicy};
 
@@ -16,6 +17,29 @@ use crate::{Error, Result};
 #[path = "./publish_test.rs"]
 mod publish_test;
 
+/// Contains label=value data for use with publishing
+#[derive(Debug, Clone)]
+pub struct PublishLabel {
+    label: String,
+    value: String,
+}
+
+impl FromStr for PublishLabel {
+    type Err = crate::Error;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        if let Some((label, value)) = s.split_once('=') {
+            return Ok(Self {
+                label: label.to_string(),
+                value: value.to_string(),
+            });
+        }
+        Err(Error::String(format!(
+            "Invalid: {s} is not in 'label=value' format"
+        )))
+    }
+}
+
 /// Manages the publishing of packages from one repo to another.
 ///
 /// Usually, the publish process moves packages from the local
@@ -26,6 +50,7 @@ pub struct Publisher {
     from: Arc<storage::RepositoryHandle>,
     to: Arc<storage::RepositoryHandle>,
     skip_source_packages: bool,
+    allow_existing_label: Option<PublishLabel>,
     force: bool,
 }
 
@@ -42,6 +67,7 @@ impl Publisher {
             from: source,
             to: destination,
             skip_source_packages: false,
+            allow_existing_label: None,
             force: false,
         }
     }
@@ -64,12 +90,39 @@ impl Publisher {
         self
     }
 
+    /// Allow publishing builds when the version already exists and
+    /// the existing version recipe contains the given metadata label
+    /// and value.
+    pub fn allow_existing_with_label(mut self, existing_label: Option<PublishLabel>) -> Self {
+        self.allow_existing_label = existing_label;
+        self
+    }
+
     /// Forcefully publishing a package will overwrite an existing publish if it exists.
     pub fn force(mut self, force: bool) -> Self {
         self.force = force;
         self
     }
 
+    async fn allow_existing_version(&self, dest_recipe_ident: &VersionIdent) -> Result<bool> {
+        if let Some(label) = &self.allow_existing_label {
+            let dest_recipe = self.to.read_recipe(dest_recipe_ident).await?;
+            let result = dest_recipe
+                .metadata()
+                .has_label_with_value(&label.label, &label.value);
+            if !result {
+                tracing::debug!(
+                    "Package version does not have '{}={}' in its metadata",
+                    label.label,
+                    label.value
+                );
+            }
+            Ok(result)
+        } else {
+            Ok(false)
+        }
+    }
+
     /// Publish the identified package as configured.
     pub async fn publish<I>(&self, pkg: I) -> Result<Vec<BuildIdent>>
     where
@@ -101,15 +154,22 @@ impl Publisher {
                         Ok(_) => {
                             // Do nothing if no errors.
                         }
-                        Err(spk_storage::Error::VersionExists(_)) => {
-                            match pkg.build() {
-                                Some(_) => (), // If build provided, we can silently fail.
-                                None => {
-                                    return Err(format!(
-                                        "Failed to publish recipe {}: Version exists",
-                                        recipe.ident(),
-                                    )
-                                    .into());
+                        Err(spk_storage::Error::VersionExists(dest_recipe_ident)) => {
+                            if self.allow_existing_version(&dest_recipe_ident).await? {
+                                // The existing version was generated and published by a
+                                // conversion process, e.g. spk convert pip/spk-convert-pip,
+                                // so allow these builds to be published.
+                                tracing::info!("Package version exists, allow-existing-with-label specified, and matched: publishing new builds allowed");
+                            } else {
+                                match pkg.build() {
+                                    Some(_) => (), // If build provided, we can silently fail.
+                                    None => {
+                                        return Err(format!(
+                                            "Failed to publish recipe {}: Version exists",
+                                            recipe.ident(),
+                                        )
+                                        .into());
+                                    }
                                 }
                             }
                         }

crates/spk-cli/group2/src/cmd_publish.rs

@@ -6,7 +6,7 @@ use std::sync::Arc;
 
 use clap::Args;
 use miette::Result;
-use spk_cli_common::{CommandArgs, Publisher, Run};
+use spk_cli_common::{CommandArgs, PublishLabel, Publisher, Run};
 use spk_schema::ident_ops::{NormalizedTagStrategy, VerbatimTagStrategy};
 use spk_schema::AnyIdent;
 use spk_storage as storage;
@@ -39,6 +39,12 @@ pub struct Publish {
     #[clap(long, short)]
     force: bool,
 
+    /// Allow publishing builds when the version already exists and
+    /// the existing version recipe contains the given metadata label
+    /// and value.
+    #[clap(long, hide = true, value_name = "LABEL=VALUE")]
+    allow_existing_with_label: Option<PublishLabel>,
+
     /// The local packages to publish
     ///
     /// This can be an entire package version with all builds or a
@@ -76,6 +82,7 @@ impl Run for Publish {
 
         let publisher = Publisher::new(Arc::new(source.into()), Arc::new(target))
             .skip_source_packages(self.no_source)
+            .allow_existing_with_label(self.allow_existing_with_label.clone())
             .force(self.force);
 
         let mut published = Vec::new();

crates/spk-schema/src/metadata/meta.rs

@@ -48,6 +48,13 @@ impl Meta {
         "Unlicensed".into()
     }
 
+    pub fn has_label_with_value(&self, label: &str, value: &str) -> bool {
+        if let Some(label_value) = self.labels.get(label) {
+            return *label_value == value;
+        }
+        false
+    }
+
     pub fn update_metadata(&mut self, global_config: &Metadata) -> Result<i32> {
         for config in global_config.global.iter() {
             let cmd = &config.command;

crates/spk-schema/src/recipe.rs

@@ -11,6 +11,7 @@ use spk_schema_ident::VersionIdent;
 use crate::foundation::ident_build::BuildId;
 use crate::foundation::option_map::OptionMap;
 use crate::foundation::spec_ops::{Named, Versioned};
+use crate::metadata::Meta;
 use crate::{InputVariant, Package, RequirementsList, Result, TestStage, Variant};
 
 /// Return the resolved packages from a solution.
@@ -79,6 +80,9 @@ pub trait Recipe:
         V: InputVariant,
         E: BuildEnv<Package = P>,
         P: Package;
+
+    /// Return the metadata for this package.
+    fn metadata(&self) -> &Meta;
 }
 
 impl<T> Recipe for std::sync::Arc<T>
@@ -137,6 +141,10 @@ where
     {
         (**self).generate_binary_build(variant, build_env)
     }
+
+    fn metadata(&self) -> &Meta {
+        (**self).metadata()
+    }
 }
 
 impl<T> Recipe for &T
@@ -195,4 +203,8 @@ where
     {
         (**self).generate_binary_build(variant, build_env)
     }
+
+    fn metadata(&self) -> &Meta {
+        (**self).metadata()
+    }
 }

crates/spk-schema/src/spec.rs

@@ -20,6 +20,7 @@ use crate::foundation::option_map::OptionMap;
 use crate::foundation::spec_ops::prelude::*;
 use crate::foundation::version::{Compat, Compatibility, Version};
 use crate::ident::{PkgRequest, Request, Satisfy, VarRequest};
+use crate::metadata::Meta;
 use crate::{
     v0,
     BuildEnv,
@@ -367,6 +368,13 @@ impl Recipe for SpecRecipe {
                 .map(Spec::V0Package),
         }
     }
+
+    fn metadata(&self) -> &Meta {
+        match self {
+            SpecRecipe::V0Package(r) => r.metadata(),
+            SpecRecipe::V0Platform(r) => r.metadata(),
+        }
+    }
 }
 
 impl Named for SpecRecipe {

crates/spk-schema/src/v0/platform.rs

@@ -284,6 +284,10 @@ impl Recipe for Platform {
 
         spec.generate_binary_build(variant, build_env)
     }
+
+    fn metadata(&self) -> &Meta {
+        &self.meta
+    }
 }
 
 // A private visitor struct that may be extended to aid linting in future.

crates/spk-schema/src/v0/spec.rs

@@ -651,6 +651,10 @@ impl Recipe for Spec<VersionIdent> {
         let digest = self.build_digest(variant.input_variant())?;
         Ok(updated.map_ident(|i| i.into_build(Build::BuildId(digest))))
     }
+
+    fn metadata(&self) -> &Meta {
+        &self.meta
+    }
 }
 
 impl Satisfy<PkgRequest> for Spec<BuildIdent> {

packages/spk-convert-pip/spk-convert-pip

@@ -31,6 +31,9 @@ TRUNCATED_VALUE_INDICATOR = "..."
 # Long license values are truncated to this number of characters to
 # allow "..." to be added to the value so show it has been truncated.
 LICENSE_FIELD_TRUNCATION_POINT = LICENSE_FIELD_LIMIT - len(TRUNCATED_VALUE_INDICATOR)
+# Metadata labels
+SPK_GENERATED_BY_LABEL = "spk:generated_by"
+SPK_GENERATED_BY_VALUE = "spk-convert-pip"
 
 
 def spk_exe() -> str:
@@ -120,7 +123,7 @@ def main() -> int:
         )
 
     if args.publish:
-        cmd = [spk_exe(), "publish", "-r", args.target_repo]
+        cmd = [spk_exe(), "publish", "--allow-existing-with-label", f"{SPK_GENERATED_BY_LABEL}={SPK_GENERATED_BY_VALUE}", "-r", args.target_repo]
         if args.force:
             cmd.append("--force")
         cmd.extend([spec["pkg"] for spec in specs])
@@ -292,7 +295,7 @@ class PipImporter:
             "meta": {
                 "license": package_license,
                 "labels": {
-                    "spk:generated_by": "spk-convert-pip",
+                    SPK_GENERATED_BY_LABEL: SPK_GENERATED_BY_VALUE,
                     "spk-convert-pip:cli": self._cli_args,
                 },
             },