From 604f0387524f34299b4ff830cf04a4c11b8ea61a Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Wed, 11 Oct 2023 16:20:01 -0600 Subject: [PATCH 1/5] feat(oci): manifest/config updates to support containerd Signed-off-by: Vaughn Dice --- Cargo.lock | 5 +- crates/oci/Cargo.toml | 2 +- crates/oci/src/client.rs | 99 ++++++++++++++++++++++++++++++++++------ 3 files changed, 87 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69616db900..baadd7a9c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3995,10 +3995,10 @@ dependencies = [ [[package]] name = "oci-distribution" version = "0.10.0" -source = "git+https://github.com/fermyon/oci-distribution?rev=05022618d78feef9b99f20b5da8fd6def6bb80d2#05022618d78feef9b99f20b5da8fd6def6bb80d2" +source = "git+https://github.com/fermyon/oci-distribution?rev=639c907b7c0c4e74716356585410d4abe4aebf4d#639c907b7c0c4e74716356585410d4abe4aebf4d" dependencies = [ + "bytes", "chrono", - "futures", "futures-util", "http", "http-auth", @@ -4012,7 +4012,6 @@ dependencies = [ "sha2", "thiserror", "tokio", - "tokio-util 0.7.9", "tracing", "unicase", ] diff --git a/crates/oci/Cargo.toml b/crates/oci/Cargo.toml index 314f4b83a6..e741a8bb14 100644 --- a/crates/oci/Cargo.toml +++ b/crates/oci/Cargo.toml @@ -14,7 +14,7 @@ dkregistry = { git = "https://github.com/camallo/dkregistry-rs", rev = "37acecb4 docker_credential = "1.0" dirs = "4.0" futures-util = "0.3" -oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "05022618d78feef9b99f20b5da8fd6def6bb80d2" } +oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "639c907b7c0c4e74716356585410d4abe4aebf4d" } reqwest = "0.11" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/oci/src/client.rs b/crates/oci/src/client.rs index f3f2d01544..f17f313bde 100644 --- a/crates/oci/src/client.rs +++ b/crates/oci/src/client.rs @@ -6,13 +6,13 @@ use anyhow::{bail, Context, Result}; use docker_credential::DockerCredential; use futures_util::future; use futures_util::stream::{self, StreamExt, TryStreamExt}; -use oci_distribution::token_cache::RegistryTokenType; -use oci_distribution::RegistryOperation; use oci_distribution::{ client::{Config, ImageLayer}, - manifest::OciImageManifest, + errors::OciDistributionError, + manifest::{OciImageManifest, OCI_IMAGE_MEDIA_TYPE}, secrets::RegistryAuth, - Reference, + token_cache::RegistryTokenType, + Reference, RegistryOperation, }; use reqwest::Url; use spin_common::sha256; @@ -25,15 +25,19 @@ use walkdir::WalkDir; use crate::auth::AuthConfig; -// TODO: the media types for application, wasm module, data and archive layer are not final. +// TODO: the media types for application, data and archive layer are not final /// Media type for a layer representing a locked Spin application configuration pub const SPIN_APPLICATION_MEDIA_TYPE: &str = "application/vnd.fermyon.spin.application.v1+config"; -// Note: we hope to use a canonical value defined upstream for this media type -const WASM_LAYER_MEDIA_TYPE: &str = "application/vnd.wasm.content.layer.v1+wasm"; /// Media type for a layer representing a generic data file used by a Spin application pub const DATA_MEDIATYPE: &str = "application/vnd.wasm.content.layer.v1+data"; /// Media type for a layer representing a compressed archive of one or more files used by a Spin application pub const ARCHIVE_MEDIATYPE: &str = "application/vnd.wasm.content.bundle.v1.tar+gzip"; +// Legacy wasm layer media type used by pre-2.0 versions of Spin +const WASM_LAYER_MEDIA_TYPE_LEGACY: &str = "application/vnd.wasm.content.layer.v1+wasm"; + +// TODO: use canonical types defined upstream; see https://github.com/bytecodealliance/registry/pull/146 +const WASM_LAYER_MEDIA_TYPE: &str = "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"; +const COMPONENT_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.component.v1+wasm"; const CONFIG_FILE: &str = "config.json"; const LATEST_TAG: &str = "latest"; @@ -164,12 +168,24 @@ impl Client { locked.components = components; locked.metadata.remove("origin"); + // Push layer for locked spin application config + let locked_config_layer = ImageLayer::new( + serde_json::to_vec(&locked).context("could not serialize locked config")?, + SPIN_APPLICATION_MEDIA_TYPE.to_string(), + None, + ); + layers.push(locked_config_layer); + let oci_config = Config { + // TODO: now that the locked config bytes are pushed as a layer, what should data here be? + // Keeping as locked config bytes would make it feasible for older Spin clients to pull/run + // apps published by newer Spin clients data: serde_json::to_vec(&locked)?, - media_type: SPIN_APPLICATION_MEDIA_TYPE.to_string(), + media_type: OCI_IMAGE_MEDIA_TYPE.to_string(), annotations: None, }; - let manifest = OciImageManifest::build(&layers, &oci_config, None); + let mut manifest = OciImageManifest::build(&layers, &oci_config, None); + manifest.artifact_type = Some(COMPONENT_ARTIFACT_TYPE.to_string()); let response = self .oci .push(&reference, &layers, oci_config, &auth, Some(manifest)) @@ -275,16 +291,17 @@ impl Client { let m = self.manifest_path(&reference.to_string()).await?; fs::write(&m, &manifest_json).await?; + // Older published Spin apps feature the locked app config *as* the OCI manifest config layer, + // while newer versions publish the locked app config as a generic layer alongside others. + // Assume that these bytes may represent the locked app config and write it as such. + // TODO: update this assumption if we change the data we write to the OCI manifest config layer. let mut cfg_bytes = Vec::new(); self.oci .pull_blob(&reference, &manifest.config.digest, &mut cfg_bytes) .await?; - let cfg = std::str::from_utf8(&cfg_bytes)?; - tracing::debug!("Pulled config: {}", cfg); - - // Write the config object in `/registry/oci/manifests/repository:/config.json` - let c = self.lockfile_path(&reference.to_string()).await?; - fs::write(&c, &cfg).await?; + self.write_locked_app_config(&reference.to_string(), &cfg_bytes) + .await + .context("unable to write locked app config to cache")?; // If a layer is a Wasm module, write it in the Wasm directory. // Otherwise, write it in the data directory (after unpacking if archive layer) @@ -298,6 +315,7 @@ impl Client { || this.cache.data_file(&layer.digest).is_ok() { tracing::debug!("Layer {} already exists in cache", &layer.digest); +<<<<<<< HEAD return anyhow::Ok(()); } @@ -315,6 +333,44 @@ impl Client { } _ => { this.cache.write_data(&bytes, &layer.digest).await?; +======= + } else { + tracing::debug!("Pulling layer {}", &layer.digest); + let mut bytes = Vec::new(); + match this + .oci + .pull_blob(&reference, &layer.digest, &mut bytes) + .await + { + Err(e) => return Err(e), + _ => match layer.media_type.as_str() { + // If the locked app config is present as a separate layer, this should take precedence + SPIN_APPLICATION_MEDIA_TYPE => { + if let Err(e) = this.write_locked_app_config(&reference.to_string(), &bytes) + .await + { + return Err(OciDistributionError::GenericError( + Some(format!("unable to write locked app config to cache: {}", e)) + )); + } + } + WASM_LAYER_MEDIA_TYPE | WASM_LAYER_MEDIA_TYPE_LEGACY => { + let _ = this.cache.write_wasm(&bytes, &layer.digest).await; + } + ARCHIVE_MEDIATYPE => { + if let Err(e) = + this.unpack_archive_layer(&bytes, &layer.digest).await + { + return Err(OciDistributionError::GenericError(Some( + format!("unable to unpack archive layer with digest {}: {}", &layer.digest, e), + ))); + } + } + _ => { + let _ = this.cache.write_data(&bytes, &layer.digest).await; + } + }, +>>>>>>> 942a1782 (feat(oci): manifest/config updates to support containerd) } } Ok(()) @@ -373,6 +429,19 @@ impl Client { Ok(p.join(CONFIG_FILE)) } + /// Write the config object in `/registry/oci/manifests/repository:/config.json` + async fn write_locked_app_config( + &self, + reference: impl AsRef, + bytes: impl AsRef<[u8]>, + ) -> Result<()> { + let cfg = std::str::from_utf8(bytes.as_ref())?; + tracing::debug!("Pulled config: {}", cfg); + + let c = self.lockfile_path(reference).await?; + fs::write(&c, &cfg).await.map_err(anyhow::Error::from) + } + /// Create a new wasm layer based on a file. async fn wasm_layer(file: &Path) -> Result { tracing::log::trace!("Reading wasm module from {:?}", file); From 50053dac4722ee17d75d62b49e2f832a137a3499 Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Tue, 17 Oct 2023 15:00:22 -0600 Subject: [PATCH 2/5] remove artifactType from OCI manifest Signed-off-by: Vaughn Dice --- crates/oci/src/client.rs | 52 +++++++--------------------------------- 1 file changed, 8 insertions(+), 44 deletions(-) diff --git a/crates/oci/src/client.rs b/crates/oci/src/client.rs index f17f313bde..0fe0d104b8 100644 --- a/crates/oci/src/client.rs +++ b/crates/oci/src/client.rs @@ -8,7 +8,6 @@ use futures_util::future; use futures_util::stream::{self, StreamExt, TryStreamExt}; use oci_distribution::{ client::{Config, ImageLayer}, - errors::OciDistributionError, manifest::{OciImageManifest, OCI_IMAGE_MEDIA_TYPE}, secrets::RegistryAuth, token_cache::RegistryTokenType, @@ -37,7 +36,6 @@ const WASM_LAYER_MEDIA_TYPE_LEGACY: &str = "application/vnd.wasm.content.layer.v // TODO: use canonical types defined upstream; see https://github.com/bytecodealliance/registry/pull/146 const WASM_LAYER_MEDIA_TYPE: &str = "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"; -const COMPONENT_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.component.v1+wasm"; const CONFIG_FILE: &str = "config.json"; const LATEST_TAG: &str = "latest"; @@ -184,8 +182,8 @@ impl Client { media_type: OCI_IMAGE_MEDIA_TYPE.to_string(), annotations: None, }; - let mut manifest = OciImageManifest::build(&layers, &oci_config, None); - manifest.artifact_type = Some(COMPONENT_ARTIFACT_TYPE.to_string()); + let manifest = OciImageManifest::build(&layers, &oci_config, None); + let response = self .oci .push(&reference, &layers, oci_config, &auth, Some(manifest)) @@ -315,7 +313,6 @@ impl Client { || this.cache.data_file(&layer.digest).is_ok() { tracing::debug!("Layer {} already exists in cache", &layer.digest); -<<<<<<< HEAD return anyhow::Ok(()); } @@ -325,7 +322,12 @@ impl Client { .pull_blob(&reference, &layer.digest, &mut bytes) .await?; match layer.media_type.as_str() { - WASM_LAYER_MEDIA_TYPE => { + SPIN_APPLICATION_MEDIA_TYPE => { + this.write_locked_app_config(&reference.to_string(), &bytes) + .await + .with_context(|| "unable to write locked app config to cache")?; + } + WASM_LAYER_MEDIA_TYPE | WASM_LAYER_MEDIA_TYPE_LEGACY => { this.cache.write_wasm(&bytes, &layer.digest).await?; } ARCHIVE_MEDIATYPE => { @@ -333,44 +335,6 @@ impl Client { } _ => { this.cache.write_data(&bytes, &layer.digest).await?; -======= - } else { - tracing::debug!("Pulling layer {}", &layer.digest); - let mut bytes = Vec::new(); - match this - .oci - .pull_blob(&reference, &layer.digest, &mut bytes) - .await - { - Err(e) => return Err(e), - _ => match layer.media_type.as_str() { - // If the locked app config is present as a separate layer, this should take precedence - SPIN_APPLICATION_MEDIA_TYPE => { - if let Err(e) = this.write_locked_app_config(&reference.to_string(), &bytes) - .await - { - return Err(OciDistributionError::GenericError( - Some(format!("unable to write locked app config to cache: {}", e)) - )); - } - } - WASM_LAYER_MEDIA_TYPE | WASM_LAYER_MEDIA_TYPE_LEGACY => { - let _ = this.cache.write_wasm(&bytes, &layer.digest).await; - } - ARCHIVE_MEDIATYPE => { - if let Err(e) = - this.unpack_archive_layer(&bytes, &layer.digest).await - { - return Err(OciDistributionError::GenericError(Some( - format!("unable to unpack archive layer with digest {}: {}", &layer.digest, e), - ))); - } - } - _ => { - let _ = this.cache.write_data(&bytes, &layer.digest).await; - } - }, ->>>>>>> 942a1782 (feat(oci): manifest/config updates to support containerd) } } Ok(()) From 6b1db2652897ae8105149a8144cb5f9fcff1d64e Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Thu, 19 Oct 2023 14:41:03 -0600 Subject: [PATCH 3/5] create default config for OCI manifest Signed-off-by: Vaughn Dice --- crates/oci/src/client.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/crates/oci/src/client.rs b/crates/oci/src/client.rs index 0fe0d104b8..e783476830 100644 --- a/crates/oci/src/client.rs +++ b/crates/oci/src/client.rs @@ -7,11 +7,8 @@ use docker_credential::DockerCredential; use futures_util::future; use futures_util::stream::{self, StreamExt, TryStreamExt}; use oci_distribution::{ - client::{Config, ImageLayer}, - manifest::{OciImageManifest, OCI_IMAGE_MEDIA_TYPE}, - secrets::RegistryAuth, - token_cache::RegistryTokenType, - Reference, RegistryOperation, + client::ImageLayer, config::ConfigFile, manifest::OciImageManifest, secrets::RegistryAuth, + token_cache::RegistryTokenType, Reference, RegistryOperation, }; use reqwest::Url; use spin_common::sha256; @@ -174,14 +171,15 @@ impl Client { ); layers.push(locked_config_layer); - let oci_config = Config { - // TODO: now that the locked config bytes are pushed as a layer, what should data here be? - // Keeping as locked config bytes would make it feasible for older Spin clients to pull/run - // apps published by newer Spin clients - data: serde_json::to_vec(&locked)?, - media_type: OCI_IMAGE_MEDIA_TYPE.to_string(), - annotations: None, + // Construct empty/default OCI config file. Data may be parsed according to + // the expected config structure per the image spec, so we want to ensure it conforms. + // (See https://github.com/opencontainers/image-spec/blob/main/config.md) + // TODO: Explore adding data applicable to a Spin app. + let oci_config_file = ConfigFile { + ..Default::default() }; + let oci_config = + oci_distribution::client::Config::oci_v1_from_config_file(oci_config_file, None)?; let manifest = OciImageManifest::build(&layers, &oci_config, None); let response = self @@ -292,7 +290,6 @@ impl Client { // Older published Spin apps feature the locked app config *as* the OCI manifest config layer, // while newer versions publish the locked app config as a generic layer alongside others. // Assume that these bytes may represent the locked app config and write it as such. - // TODO: update this assumption if we change the data we write to the OCI manifest config layer. let mut cfg_bytes = Vec::new(); self.oci .pull_blob(&reference, &manifest.config.digest, &mut cfg_bytes) From dd071444a4874351f6c2548e33e16eb5d762e133 Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Fri, 20 Oct 2023 16:29:52 -0600 Subject: [PATCH 4/5] bump oci-distribution rev to get Os::Wasip1; update OCI config with arch and os Signed-off-by: Vaughn Dice --- Cargo.lock | 2 +- crates/oci/Cargo.toml | 2 +- crates/oci/src/client.rs | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index baadd7a9c7..0b3be75437 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3995,7 +3995,7 @@ dependencies = [ [[package]] name = "oci-distribution" version = "0.10.0" -source = "git+https://github.com/fermyon/oci-distribution?rev=639c907b7c0c4e74716356585410d4abe4aebf4d#639c907b7c0c4e74716356585410d4abe4aebf4d" +source = "git+https://github.com/fermyon/oci-distribution?rev=63cbb0925775e0c9c870195cad1d50ac8707a264#63cbb0925775e0c9c870195cad1d50ac8707a264" dependencies = [ "bytes", "chrono", diff --git a/crates/oci/Cargo.toml b/crates/oci/Cargo.toml index e741a8bb14..33dfa5d90f 100644 --- a/crates/oci/Cargo.toml +++ b/crates/oci/Cargo.toml @@ -14,7 +14,7 @@ dkregistry = { git = "https://github.com/camallo/dkregistry-rs", rev = "37acecb4 docker_credential = "1.0" dirs = "4.0" futures-util = "0.3" -oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "639c907b7c0c4e74716356585410d4abe4aebf4d" } +oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "63cbb0925775e0c9c870195cad1d50ac8707a264" } reqwest = "0.11" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/oci/src/client.rs b/crates/oci/src/client.rs index e783476830..55f1e833ed 100644 --- a/crates/oci/src/client.rs +++ b/crates/oci/src/client.rs @@ -174,8 +174,10 @@ impl Client { // Construct empty/default OCI config file. Data may be parsed according to // the expected config structure per the image spec, so we want to ensure it conforms. // (See https://github.com/opencontainers/image-spec/blob/main/config.md) - // TODO: Explore adding data applicable to a Spin app. + // TODO: Explore adding data applicable to the Spin app being published. let oci_config_file = ConfigFile { + architecture: oci_distribution::config::Architecture::Wasm, + os: oci_distribution::config::Os::Wasip1, ..Default::default() }; let oci_config = From 464214bbc0335d64ec12c67a30ee0cd4967a3370 Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Wed, 25 Oct 2023 11:05:30 -0600 Subject: [PATCH 5/5] keep WASM_LAYER_MEDIA_TYPE the same until canonical value finalized Signed-off-by: Vaughn Dice --- crates/oci/src/client.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/oci/src/client.rs b/crates/oci/src/client.rs index 55f1e833ed..ffa1ce86c3 100644 --- a/crates/oci/src/client.rs +++ b/crates/oci/src/client.rs @@ -28,11 +28,8 @@ pub const SPIN_APPLICATION_MEDIA_TYPE: &str = "application/vnd.fermyon.spin.appl pub const DATA_MEDIATYPE: &str = "application/vnd.wasm.content.layer.v1+data"; /// Media type for a layer representing a compressed archive of one or more files used by a Spin application pub const ARCHIVE_MEDIATYPE: &str = "application/vnd.wasm.content.bundle.v1.tar+gzip"; -// Legacy wasm layer media type used by pre-2.0 versions of Spin -const WASM_LAYER_MEDIA_TYPE_LEGACY: &str = "application/vnd.wasm.content.layer.v1+wasm"; - -// TODO: use canonical types defined upstream; see https://github.com/bytecodealliance/registry/pull/146 -const WASM_LAYER_MEDIA_TYPE: &str = "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"; +// Note: this will be updated with a canonical value once defined upstream +const WASM_LAYER_MEDIA_TYPE: &str = "application/vnd.wasm.content.layer.v1+wasm"; const CONFIG_FILE: &str = "config.json"; const LATEST_TAG: &str = "latest"; @@ -326,7 +323,7 @@ impl Client { .await .with_context(|| "unable to write locked app config to cache")?; } - WASM_LAYER_MEDIA_TYPE | WASM_LAYER_MEDIA_TYPE_LEGACY => { + WASM_LAYER_MEDIA_TYPE => { this.cache.write_wasm(&bytes, &layer.digest).await?; } ARCHIVE_MEDIATYPE => {