From a72bd3cda93e6f2d9411e743ec5d5c4fd56b5721 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Tue, 5 Aug 2025 14:22:26 +0200 Subject: [PATCH 01/47] docs: ADR for re-processing of documents --- docs/adrs/00008-re-process-documents.md | 81 +++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/adrs/00008-re-process-documents.md diff --git a/docs/adrs/00008-re-process-documents.md b/docs/adrs/00008-re-process-documents.md new file mode 100644 index 000000000..db9874def --- /dev/null +++ b/docs/adrs/00008-re-process-documents.md @@ -0,0 +1,81 @@ +# 00008. Re-process documents + +Date: 2025-08-08 + +## Status + +DRAFT + +## Context + +During the process of ingestion, we extract certain information of the uploaded documents and store that information +in the database. We also store the original source document "as-is". + +When making changes to the database structure, we also have a migration process, which takes care of upgrading the +database structures during an upgrade. + +However, in some cases, changing the database structure actually means to extract more information from documents and is +currently stored in the database. Or information is extracted in a different way. This requires a re-processing of +all documents affected by this change. + +### Example + +We do ignore all CVSS v2 scores at the moment. Adding new fields for storing v2 scores, we wouldn't have +any stored in the database without re-processing documents and extracting that information. + +### Assumptions + +This ADR makes the following assumptions: + +* All documents are stored in the storage +* It is expected that an upgrade is actually required +* Running such migrations is expected to take a long time + +Question? Do we want to support downgrades? + +## Decision + +### Option 1 + +During the migration of database structures (sea orm), we also re-process all documents (when required). + +In order to report progress, we could write that state into a table and expose that information to the user via the UI. + +* 👎 Might serve inaccurate data for a while +* 👎 Might block an upgrade if re-processing fails +* 👍 Can fully migrate database (create mandatory field as optional -> re-process -> make mandatory) +* 👎 Might be tricky to create a combined re-processing of multiple ones + +### Option 2 + +We create a similar module as for the importer. Running migrations after an upgrade. Accepting that in the meantime, +we might service inaccurate data. + +* 👎 Might serve inaccurate data for a while for a longer time +* 👎 Can't fully migrate database (new mandatory field won't work) +* 👍 Upgrade process is faster and less complex +* 👎 Requires some coordination between instances (only one processor at a time, maybe one after the other) + +### Option 3 + +We change ingestion in a way to it is possible to just re-ingest every document. Meaning, we re-ingest from the +original sources. + +* 👎 Might serve inaccurate data for a while for a longer time +* 👎 Can't fully migrate database (new mandatory field won't work) +* 👍 Upgrade process is faster and less complex +* 👎 Original sources might no longer have the documents +* 👎 Won't work for manual (API) uploads +* 👎 Would require removing optimizations for existing documents + +## Open items + +… + +## Alternative approaches + +… + +## Consequences + +… From 134611051b8eaf153fd35681cf71730076002ff3 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 6 Aug 2025 15:56:56 +0200 Subject: [PATCH 02/47] chore: initial PoC impl --- Cargo.lock | 7 + migration/Cargo.toml | 13 +- migration/src/data/mod.rs | 159 ++++++++++++++++++ migration/src/lib.rs | 4 + .../src/m0002000_example_data_migration.rs | 52 ++++++ modules/storage/src/service/dispatch.rs | 6 +- modules/storage/src/service/fs.rs | 4 +- modules/storage/src/service/mod.rs | 15 +- modules/storage/src/service/s3.rs | 6 +- 9 files changed, 251 insertions(+), 15 deletions(-) create mode 100644 migration/src/data/mod.rs create mode 100644 migration/src/m0002000_example_data_migration.rs diff --git a/Cargo.lock b/Cargo.lock index 2c6b26143..2691de3cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8401,7 +8401,13 @@ name = "trustify-migration" version = "0.4.0-beta.1" dependencies = [ "anyhow", + "bytes", + "futures-util", + "sea-orm", "sea-orm-migration", + "serde-cyclonedx", + "serde_json", + "spdx-rs", "test-context", "test-log", "tokio", @@ -8409,6 +8415,7 @@ dependencies = [ "trustify-common", "trustify-db", "trustify-entity", + "trustify-module-storage", "trustify-test-context", "uuid", ] diff --git a/migration/Cargo.toml b/migration/Cargo.toml index 9a38e3a98..08160bd6f 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -10,13 +10,24 @@ name = "migration" path = "src/lib.rs" [dependencies] +trustify-common = { workspace = true } +trustify-entity = { workspace = true } +trustify-module-storage = { workspace = true } + +bytes = { workspace = true } +futures-util = { workspace = true } +anyhow = { workspace = true } +sea-orm = { workspace = true } sea-orm-migration = { workspace = true, features = ["runtime-tokio-rustls", "sqlx-postgres", "with-uuid"] } +serde-cyclonedx = { workspace = true } +serde_json = { workspace = true } +spdx-rs = { workspace = true } tokio = { workspace = true, features = ["full"] } uuid = { workspace = true, features = ["v5"] } [dev-dependencies] trustify-common = { workspace = true } -trustify-db = { workspace = true } +trustify-db = { workspace = true } trustify-entity = { workspace = true } trustify-test-context = { workspace = true } diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs new file mode 100644 index 000000000..1d094abee --- /dev/null +++ b/migration/src/data/mod.rs @@ -0,0 +1,159 @@ +use anyhow::{anyhow, bail}; +use bytes::BytesMut; +use futures_util::stream::TryStreamExt; +use futures_util::{StreamExt, stream}; +use sea_orm::{ + ConnectionTrait, DatabaseTransaction, DbErr, EntityTrait, ModelTrait, TransactionTrait, +}; +use sea_orm_migration::SchemaManager; +use trustify_common::id::Id; +use trustify_entity::{sbom, source_document}; +use trustify_module_storage::service::{StorageBackend, StorageKey, dispatch::DispatchBackend}; + +#[allow(clippy::large_enum_variant)] +pub enum Sbom { + CycloneDx(serde_cyclonedx::cyclonedx::v_1_6::CycloneDx), + Spdx(spdx_rs::models::SPDX), +} + +pub trait Document: Sized + Send + Sync { + type Model: Send; + + async fn all(tx: &C) -> Result, DbErr> + where + C: ConnectionTrait; + + async fn source(model: &Self::Model, storage: &S, tx: &C) -> Result + where + S: StorageBackend + Send + Sync, + C: ConnectionTrait; +} + +impl Document for Sbom { + type Model = sbom::Model; + + async fn all(tx: &C) -> Result, DbErr> { + sbom::Entity::find().all(tx).await + } + + async fn source(model: &Self::Model, storage: &S, tx: &C) -> Result + where + S: StorageBackend + Send + Sync, + C: ConnectionTrait, + { + let source = model.find_related(source_document::Entity).one(tx).await?; + + let Some(source) = source else { + bail!("Missing source document ID for SBOM: {}", model.sbom_id); + }; + + let stream = storage + .retrieve( + StorageKey::try_from(Id::Sha256(source.sha256)) + .map_err(|err| anyhow!("Invalid ID: {err}"))?, + ) + .await + .map_err(|err| anyhow!("Failed to retrieve document: {err}"))? + .ok_or_else(|| anyhow!("Missing source document for SBOM: {}", model.sbom_id))?; + + stream + .try_collect::() + .await + .map_err(|err| anyhow!("Failed to collect bytes: {err}")) + .map(|bytes| bytes.freeze()) + .and_then(|bytes| { + serde_json::from_slice(&bytes) + .map(Sbom::Spdx) + .or_else(|_| serde_json::from_slice(&bytes).map(Sbom::CycloneDx)) + .map_err(|err| anyhow!("Failed to parse document: {err}")) + }) + } +} + +pub trait Handler: Send +where + D: Document, +{ + async fn call( + &self, + document: D, + model: D::Model, + tx: &DatabaseTransaction, + ) -> anyhow::Result<()>; +} + +pub trait DocumentProcessor { + async fn process( + &self, + storage: &DispatchBackend, + f: impl Handler, + ) -> anyhow::Result<(), DbErr> + where + D: Document; +} + +impl<'c> DocumentProcessor for SchemaManager<'c> { + async fn process( + &self, + storage: &DispatchBackend, + f: impl Handler, + ) -> anyhow::Result<(), DbErr> + where + D: Document, + { + let db = self.get_connection(); + let tx = db.begin().await?; + + // TODO: soft-lock database + // In order to prevent new documents with an old version to be created in the meantime, we + // should soft-lock the database. + + let all = D::all(&tx).await?; + + stream::iter(all) + .map(async |model| { + let doc = D::source(&model, storage, &tx).await.map_err(|err| { + DbErr::Migration(format!("Failed to load source document: {err}")) + })?; + f.call(doc, model, &tx).await.map_err(|err| { + DbErr::Migration(format!("Failed to process document: {err}")) + })?; + + Ok::<_, DbErr>(()) + }) + .buffer_unordered(10) // TODO: make this configurable + .try_collect::>() + .await?; + + // TODO: soft-unlock database + + Ok(()) + } +} + +#[macro_export] +macro_rules! handler { + (async | $doc:ident: $doc_ty:ty, $model:ident, $tx:ident | $body:block) => {{ + struct H; + + impl $crate::data::Handler<$doc_ty> for H { + async fn call( + &self, + $doc: $doc_ty, + $model: <$doc_ty as $crate::data::Document>::Model, + $tx: &sea_orm::DatabaseTransaction, + ) -> anyhow::Result<()> { + $body + } + } + + H + }}; +} + +#[macro_export] +macro_rules! sbom { + (async | $doc:ident, $model:ident, $tx:ident | $body:block) => { + $crate::handler!(async |$doc: $crate::data::Sbom, $model, $tx| $body) + }; +} diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 2592b0bfc..767d83883 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -1,5 +1,7 @@ pub use sea_orm_migration::prelude::*; +mod data; + mod m0000010_init; mod m0000020_add_sbom_group; mod m0000030_perf_adv_vuln; @@ -32,6 +34,7 @@ mod m0001170_non_null_source_document_id; mod m0001180_expand_spdx_licenses_with_mappings_function; mod m0001190_optimize_product_advisory_query; mod m0001200_source_document_fk_indexes; +mod m0002000_example_data_migration; pub struct Migrator; @@ -71,6 +74,7 @@ impl MigratorTrait for Migrator { Box::new(m0001180_expand_spdx_licenses_with_mappings_function::Migration), Box::new(m0001190_optimize_product_advisory_query::Migration), Box::new(m0001200_source_document_fk_indexes::Migration), + Box::new(m0002000_example_data_migration::Migration), ] } } diff --git a/migration/src/m0002000_example_data_migration.rs b/migration/src/m0002000_example_data_migration.rs new file mode 100644 index 000000000..c61759b7b --- /dev/null +++ b/migration/src/m0002000_example_data_migration.rs @@ -0,0 +1,52 @@ +use crate::{ + data::{DocumentProcessor, Sbom}, + sbom, + sea_orm::{ActiveModelTrait, IntoActiveModel, Set}, +}; +use sea_orm_migration::prelude::*; +use trustify_module_storage::service::{dispatch::DispatchBackend, fs::FileSystemBackend}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // TODO: make this configurable + let (storage, _tmp) = FileSystemBackend::for_test() + .await + .map_err(|err| DbErr::Migration(format!("failed to create storage backend: {err}")))?; + let storage = DispatchBackend::Filesystem(storage); + + // process data + + manager + .process( + &storage, + sbom!(async |sbom, model, tx| { + let mut model = model.into_active_model(); + match sbom { + Sbom::CycloneDx(_sbom) => { + // TODO: just an example + model.authors = Set(vec![]); + } + Sbom::Spdx(_sbom) => { + // TODO: just an example + model.authors = Set(vec![]); + } + } + + model.save(tx).await?; + + Ok(()) + }), + ) + .await?; + + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + Ok(()) + } +} diff --git a/modules/storage/src/service/dispatch.rs b/modules/storage/src/service/dispatch.rs index d59e8f193..039efcf03 100644 --- a/modules/storage/src/service/dispatch.rs +++ b/modules/storage/src/service/dispatch.rs @@ -22,7 +22,7 @@ impl StorageBackend for DispatchBackend { async fn store(&self, stream: S) -> Result> where - S: AsyncRead + Unpin, + S: AsyncRead + Unpin + Send, { match self { Self::Filesystem(backend) => backend.store(stream).await.map_err(Self::map_err), @@ -30,10 +30,10 @@ impl StorageBackend for DispatchBackend { } } - async fn retrieve<'a>( + async fn retrieve( &self, key: StorageKey, - ) -> Result> + 'a>, Self::Error> + ) -> Result> + use<>>, Self::Error> where Self: Sized, { diff --git a/modules/storage/src/service/fs.rs b/modules/storage/src/service/fs.rs index c84396c51..bf9820276 100644 --- a/modules/storage/src/service/fs.rs +++ b/modules/storage/src/service/fs.rs @@ -159,10 +159,10 @@ impl StorageBackend for FileSystemBackend { Ok(result) } - async fn retrieve<'a>( + async fn retrieve( &self, key: StorageKey, - ) -> Result> + 'a>, Self::Error> { + ) -> Result> + use<>>, Self::Error> { match self.locate(key).await? { Some((path, compression)) => File::open(&path) .await diff --git a/modules/storage/src/service/mod.rs b/modules/storage/src/service/mod.rs index 46ccc4fdb..460694cb0 100644 --- a/modules/storage/src/service/mod.rs +++ b/modules/storage/src/service/mod.rs @@ -85,23 +85,26 @@ impl StorageResult { } pub trait StorageBackend { - type Error: Debug; + type Error: Debug + Display; /// Store the content from a stream fn store( &self, stream: S, - ) -> impl Future>> + ) -> impl Future>> + Send where - S: AsyncRead + Unpin; + S: AsyncRead + Unpin + Send; /// Retrieve the content as an async reader - fn retrieve<'a>( + fn retrieve( &self, key: StorageKey, ) -> impl Future< - Output = Result> + 'a>, Self::Error>, - >; + Output = Result< + Option> + Send + use>, + Self::Error, + >, + > + Send; /// Delete the stored content. /// diff --git a/modules/storage/src/service/s3.rs b/modules/storage/src/service/s3.rs index 7758ab5b0..9eeb236b7 100644 --- a/modules/storage/src/service/s3.rs +++ b/modules/storage/src/service/s3.rs @@ -135,7 +135,7 @@ impl StorageBackend for S3Backend { #[instrument(skip(self, stream), err(Debug, level=tracing::Level::INFO))] async fn store(&self, stream: S) -> Result> where - S: AsyncRead + Unpin, + S: AsyncRead + Unpin + Send, { let file = TempFile::with_compression(stream, self.compression).await?; let result = file.to_result(); @@ -163,10 +163,10 @@ impl StorageBackend for S3Backend { Ok(result) } - async fn retrieve<'a>( + async fn retrieve( &self, StorageKey(key): StorageKey, - ) -> Result> + 'a>, Self::Error> { + ) -> Result> + use<>>, Self::Error> { let req = self.client.get_object().bucket(&self.bucket).key(&key); match req.send().await { From 73b539303c6112278a72d6a004f71815119a7df4 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Thu, 7 Aug 2025 10:59:33 +0200 Subject: [PATCH 03/47] docs: refine option one as the preferred one --- docs/adrs/00008-re-process-documents.md | 63 ++++++++++++++++++------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/docs/adrs/00008-re-process-documents.md b/docs/adrs/00008-re-process-documents.md index db9874def..71db27109 100644 --- a/docs/adrs/00008-re-process-documents.md +++ b/docs/adrs/00008-re-process-documents.md @@ -14,8 +14,8 @@ in the database. We also store the original source document "as-is". When making changes to the database structure, we also have a migration process, which takes care of upgrading the database structures during an upgrade. -However, in some cases, changing the database structure actually means to extract more information from documents and is -currently stored in the database. Or information is extracted in a different way. This requires a re-processing of +However, in some cases, changing the database structure actually means extracting more information from documents than +is currently stored in the database. Or information is extracted in a different way. This requires a re-processing of all documents affected by this change. ### Example @@ -30,25 +30,58 @@ This ADR makes the following assumptions: * All documents are stored in the storage * It is expected that an upgrade is actually required * Running such migrations is expected to take a long time +* The management of infrastructure (PostgreSQL) is not in the scope of Trustify Question? Do we want to support downgrades? ## Decision -### Option 1 +During the migration of database structures (sea orm), we also re-process all documents (if required). This would +be running during the migration job of the Helm chart and would have an impact on updates as the rollout of newer +version pods would be delayed until the migration (of data) has been finished. -During the migration of database structures (sea orm), we also re-process all documents (when required). +This would also require to prevent users from creating new documents during that time. Otherwise, we would need to +re-process documents ingested during the migration time. A way of doing this could be to leverage PostgreSQL's ability +to switch into read-only mode. Having mutable operations fail with a 503 (Service Unavailable) error. This would also +allow for easy A/B (green/blue) database setups. Switching the main one to read-only, having the other one run the +migration. -In order to report progress, we could write that state into a table and expose that information to the user via the UI. +We could provide an endpoint to the UI, reporting the fact that the system is in read-only mode during a migration. -* 👎 Might serve inaccurate data for a while -* 👎 Might block an upgrade if re-processing fails * 👍 Can fully migrate database (create mandatory field as optional -> re-process -> make mandatory) +* 👍 Might allow for an out-of-band migration of data, before running the upgrade (even on a staging env) +* 👍 Would allow to continue serving data while the process is running * 👎 Might be tricky to create a combined re-processing of multiple ones +* 👎 Might block an upgrade if re-processing fails + +We do want to support different approaches of this migration. Depending on the needs of the user, the size of the +data store and the infrastructure used. + +### Approach 1 + +The "lazy" approach, where the user just runs the migration (or the new version of the application with migrations +enabled). The process will migrate schema and data. This might block the startup for a bit. But would be fast and +simple for small systems. + +### Approach 2 + +The user uses a green/blue deployment. Switching the application to use green and run migrations against blue. Once +the migrations are complete, switching back to blue. Green will be read-only and mutable API calls will fail with a 503 +error. + +An alternative to this could also be to configure the system first to go into "read-only mode", by using a default +transaction mode of read-only. + +## Open items + +* [ ] How to handle unparsable or failing documents during migration? +* [ ] Add a version number to the document, tracking upgrades + +## Alternative approaches ### Option 2 -We create a similar module as for the importer. Running migrations after an upgrade. Accepting that in the meantime, +We create a similar module as for the importer. Running migrations after an upgrade. Accepting that in the meantime, we might service inaccurate data. * 👎 Might serve inaccurate data for a while for a longer time @@ -58,7 +91,7 @@ we might service inaccurate data. ### Option 3 -We change ingestion in a way to it is possible to just re-ingest every document. Meaning, we re-ingest from the +We change ingestion in a way to it is possible to just re-ingest every document. Meaning, we re-ingest from the original sources. * 👎 Might serve inaccurate data for a while for a longer time @@ -68,14 +101,10 @@ original sources. * 👎 Won't work for manual (API) uploads * 👎 Would require removing optimizations for existing documents -## Open items - -… - -## Alternative approaches - -… ## Consequences -… +* The migration will block the upgrade process until it is finished +* Ansible and the operator will need to handle this as well +* The system will become read-only during a migration +* The UI needs to provide a page for monitoring the migration state. The backend needs to provide appropriate APIs. From f8800faa260d0df26ae3207789972b70b0e76691 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Tue, 9 Sep 2025 16:25:42 +0200 Subject: [PATCH 04/47] refactor: pull out common code for creating a storage backend --- modules/storage/src/config.rs | 41 +++++++++++++++++++++++++--- server/src/profile/api.rs | 49 +++++++++------------------------- server/src/profile/importer.rs | 22 ++------------- 3 files changed, 52 insertions(+), 60 deletions(-) diff --git a/modules/storage/src/config.rs b/modules/storage/src/config.rs index 91fa1ff42..67419123c 100644 --- a/modules/storage/src/config.rs +++ b/modules/storage/src/config.rs @@ -1,6 +1,12 @@ -use crate::service::Compression; -use std::fmt::{Display, Formatter}; -use std::path::PathBuf; +use crate::service::{ + Compression, dispatch::DispatchBackend, fs::FileSystemBackend, s3::S3Backend, +}; +use anyhow::Context; +use std::{ + fmt::{Display, Formatter}, + fs::create_dir_all, + path::PathBuf, +}; #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, clap::ValueEnum)] pub enum StorageStrategy { @@ -17,7 +23,7 @@ impl Display for StorageStrategy { } } -#[derive(clap::Args, Debug, Clone)] +#[derive(clap::Parser, Debug, Clone)] #[command(next_help_heading = "Storage")] pub struct StorageConfig { #[arg( @@ -51,6 +57,33 @@ pub struct StorageConfig { pub s3_config: S3Config, } +impl StorageConfig { + /// Create a storage backend from a storage config + pub async fn into_storage(self, devmode: bool) -> anyhow::Result { + Ok(match self.storage_strategy { + StorageStrategy::Fs => { + let storage = self + .fs_path + .as_ref() + .cloned() + .unwrap_or_else(|| PathBuf::from("./.trustify/storage")); + if devmode { + create_dir_all(&storage).context(format!( + "Failed to create filesystem storage directory: {:?}", + self.fs_path + ))?; + } + DispatchBackend::Filesystem( + FileSystemBackend::new(storage, self.compression).await?, + ) + } + StorageStrategy::S3 => { + DispatchBackend::S3(S3Backend::new(self.s3_config, self.compression).await?) + } + }) + } +} + #[derive(Clone, Debug, Default, clap::Args)] #[command(next_help_heading = "S3")] #[group(id = "s3", requires = "storage-strategy")] diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index c0fb9af49..695bf8ca4 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -6,10 +6,9 @@ use actix_web::middleware; use crate::{endpoints, profile::spawn_db_check, sample_data}; use actix_web::web; -use anyhow::Context; use bytesize::ByteSize; use futures::FutureExt; -use std::{env, fs::create_dir_all, path::PathBuf, process::ExitCode, sync::Arc}; +use std::{env, process::ExitCode, sync::Arc}; use trustify_auth::{ auth::AuthConfigArguments, authenticator::Authenticator, @@ -29,10 +28,7 @@ use trustify_infrastructure::{ }; use trustify_module_analysis::{config::AnalysisConfig, service::AnalysisService}; use trustify_module_ingestor::graph::Graph; -use trustify_module_storage::{ - config::{StorageConfig, StorageStrategy}, - service::{dispatch::DispatchBackend, fs::FileSystemBackend, s3::S3Backend}, -}; +use trustify_module_storage::{config::StorageConfig, service::dispatch::DispatchBackend}; use trustify_module_ui::{UI, endpoints::UiResources}; use utoipa::openapi::{Info, License}; @@ -144,10 +140,12 @@ mod default { #[group(id = "ui")] pub struct UiConfig { /// Issuer URL used by the UI - #[arg(id = "ui-issuer-url", long, env = "UI_ISSUER_URL", default_value_t = ISSUER_URL.to_string())] + #[arg(id = "ui-issuer-url", long, env = "UI_ISSUER_URL", default_value_t = ISSUER_URL.to_string() + )] pub issuer_url: String, /// Client ID used by the UI - #[arg(id = "ui-client-id", long, env = "UI_CLIENT_ID", default_value_t = FRONTEND_CLIENT_ID.to_string())] + #[arg(id = "ui-client-id", long, env = "UI_CLIENT_ID", default_value_t = FRONTEND_CLIENT_ID.to_string() + )] pub client_id: String, /// Scopes to request #[arg(id = "ui-scope", long, env = "UI_SCOPE", default_value = "openid")] @@ -245,28 +243,7 @@ impl InitData { .register("database", spawn_db_check(db.clone())?) .await; - let storage = match run.storage.storage_strategy { - StorageStrategy::Fs => { - let storage = run - .storage - .fs_path - .as_ref() - .cloned() - .unwrap_or_else(|| PathBuf::from("./.trustify/storage")); - if run.devmode { - create_dir_all(&storage).context(format!( - "Failed to create filesystem storage directory: {:?}", - run.storage.fs_path - ))?; - } - DispatchBackend::Filesystem( - FileSystemBackend::new(storage, run.storage.compression).await?, - ) - } - StorageStrategy::S3 => DispatchBackend::S3( - S3Backend::new(run.storage.s3_config, run.storage.compression).await?, - ), - }; + let storage = run.storage.into_storage(run.devmode).await?; let ui = UI { version: env!("CARGO_PKG_VERSION").to_string(), @@ -499,12 +476,12 @@ mod test { let context = InitContext::default(); let run = Run::from_arg_matches(&Run::augment_args(Command::new("cmd")).get_matches_from( vec![ - "cmd", - "--db-name", - "test", - "--db-port", - &ctx.postgresql.as_ref().expect("database").settings().port.to_string(), - ], + "cmd", + "--db-name", + "test", + "--db-port", + &ctx.postgresql.as_ref().expect("database").settings().port.to_string(), + ], ))?; InitData::new(context, run).await.map(|_| ()) } diff --git a/server/src/profile/importer.rs b/server/src/profile/importer.rs index ad0f50f85..ed92506e0 100644 --- a/server/src/profile/importer.rs +++ b/server/src/profile/importer.rs @@ -4,10 +4,7 @@ use std::{path::PathBuf, process::ExitCode}; use trustify_common::{config::Database, db}; use trustify_infrastructure::{Infrastructure, InfrastructureConfig, InitContext}; use trustify_module_importer::server::importer; -use trustify_module_storage::{ - config::{StorageConfig, StorageStrategy}, - service::{dispatch::DispatchBackend, fs::FileSystemBackend, s3::S3Backend}, -}; +use trustify_module_storage::{config::StorageConfig, service::dispatch::DispatchBackend}; /// Run the API server #[derive(clap::Args, Debug)] @@ -73,22 +70,7 @@ impl InitData { .register("database", spawn_db_check(db.clone())?) .await; - let storage = match run.storage.storage_strategy { - StorageStrategy::Fs => { - let storage = run - .storage - .fs_path - .as_ref() - .cloned() - .unwrap_or_else(|| PathBuf::from("./.trustify/storage")); - DispatchBackend::Filesystem( - FileSystemBackend::new(storage, run.storage.compression).await?, - ) - } - StorageStrategy::S3 => DispatchBackend::S3( - S3Backend::new(run.storage.s3_config, run.storage.compression).await?, - ), - }; + let storage = run.storage.into_storage(false).await?; Ok(InitData { db, From 0be9473e8a23ba9ec74dab9b76c165ba7cd3cdf4 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Tue, 9 Sep 2025 16:26:30 +0200 Subject: [PATCH 05/47] feat: allow running data migrations as part of migrations --- Cargo.lock | 1 + migration/Cargo.toml | 3 +- migration/src/data/migration.rs | 107 ++++++++++++++++++ migration/src/data/mod.rs | 18 +-- migration/src/lib.rs | 5 +- .../src/m0002000_example_data_migration.rs | 48 +++----- trustd/src/db.rs | 11 +- 7 files changed, 147 insertions(+), 46 deletions(-) create mode 100644 migration/src/data/migration.rs diff --git a/Cargo.lock b/Cargo.lock index 2691de3cf..2735ce1a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8402,6 +8402,7 @@ version = "0.4.0-beta.1" dependencies = [ "anyhow", "bytes", + "clap", "futures-util", "sea-orm", "sea-orm-migration", diff --git a/migration/Cargo.toml b/migration/Cargo.toml index 08160bd6f..e54ec5ff8 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -14,9 +14,10 @@ trustify-common = { workspace = true } trustify-entity = { workspace = true } trustify-module-storage = { workspace = true } +anyhow = { workspace = true } bytes = { workspace = true } +clap = { workspace = true, features = ["derive", "env"] } futures-util = { workspace = true } -anyhow = { workspace = true } sea-orm = { workspace = true } sea-orm-migration = { workspace = true, features = ["runtime-tokio-rustls", "sqlx-postgres", "with-uuid"] } serde-cyclonedx = { workspace = true } diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs new file mode 100644 index 000000000..0262d15dd --- /dev/null +++ b/migration/src/data/migration.rs @@ -0,0 +1,107 @@ +use crate::{ + async_trait, + data::{Document, DocumentProcessor, Handler}, +}; +use clap::Parser; +use sea_orm::DbErr; +use sea_orm_migration::{MigrationName, MigrationTrait, SchemaManager}; +use std::{ffi::OsString, sync::LazyLock}; +use trustify_module_storage::{config::StorageConfig, service::dispatch::DispatchBackend}; + +pub struct MigrationWithData +where + M: MigrationTraitWithData, +{ + pub storage: DispatchBackend, + pub migration: M, +} + +static STORAGE: LazyLock = LazyLock::new(init_storage); + +#[allow(clippy::expect_used)] +fn init_storage() -> DispatchBackend { + // create from env-vars only + let config = StorageConfig::parse_from::<_, OsString>(vec![]); + + tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(async { + config + .into_storage(false) + .await + .expect("Failed to create storage") + }) + }) +} + +impl MigrationWithData +where + M: MigrationTraitWithData, +{ + #[allow(clippy::expect_used)] + pub fn new(migration: M) -> Self { + Self { + storage: STORAGE.clone(), + migration, + } + } +} + +impl From for MigrationWithData +where + M: MigrationTraitWithData, +{ + fn from(value: M) -> Self { + MigrationWithData::new(value) + } +} + +pub struct SchemaDataManager<'c> { + pub manager: &'c SchemaManager<'c>, + storage: &'c DispatchBackend, +} + +impl<'c> SchemaDataManager<'c> { + pub fn new(manager: &'c SchemaManager<'c>, storage: &'c DispatchBackend) -> Self { + Self { manager, storage } + } + + pub async fn process(&self, f: impl Handler) -> Result<(), DbErr> + where + D: Document, + { + self.manager.process(self.storage, f).await + } +} + +#[async_trait::async_trait] +pub trait MigrationTraitWithData { + async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr>; + async fn down(&self, manager: &SchemaDataManager) -> Result<(), DbErr>; +} + +#[async_trait::async_trait] +impl MigrationTrait for MigrationWithData +where + M: MigrationTraitWithData + MigrationName + Send + Sync, +{ + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + self.migration + .up(&SchemaDataManager::new(manager, &self.storage)) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + self.migration + .down(&SchemaDataManager::new(manager, &self.storage)) + .await + } +} + +impl MigrationName for MigrationWithData +where + M: MigrationTraitWithData + MigrationName + Send + Sync, +{ + fn name(&self) -> &str { + self.migration.name() + } +} diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 1d094abee..27da13ad6 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -1,7 +1,13 @@ +mod migration; + +pub use migration::*; + use anyhow::{anyhow, bail}; use bytes::BytesMut; -use futures_util::stream::TryStreamExt; -use futures_util::{StreamExt, stream}; +use futures_util::{ + StreamExt, + stream::{self, TryStreamExt}, +}; use sea_orm::{ ConnectionTrait, DatabaseTransaction, DbErr, EntityTrait, ModelTrait, TransactionTrait, }; @@ -16,6 +22,7 @@ pub enum Sbom { Spdx(spdx_rs::models::SPDX), } +#[allow(async_fn_in_trait)] pub trait Document: Sized + Send + Sync { type Model: Send; @@ -70,6 +77,7 @@ impl Document for Sbom { } } +#[allow(async_fn_in_trait)] pub trait Handler: Send where D: Document, @@ -93,11 +101,7 @@ pub trait DocumentProcessor { } impl<'c> DocumentProcessor for SchemaManager<'c> { - async fn process( - &self, - storage: &DispatchBackend, - f: impl Handler, - ) -> anyhow::Result<(), DbErr> + async fn process(&self, storage: &DispatchBackend, f: impl Handler) -> Result<(), DbErr> where D: Document, { diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 767d83883..7973de697 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -1,6 +1,7 @@ pub use sea_orm_migration::prelude::*; mod data; +use crate::data::MigrationWithData; mod m0000010_init; mod m0000020_add_sbom_group; @@ -74,7 +75,9 @@ impl MigratorTrait for Migrator { Box::new(m0001180_expand_spdx_licenses_with_mappings_function::Migration), Box::new(m0001190_optimize_product_advisory_query::Migration), Box::new(m0001200_source_document_fk_indexes::Migration), - Box::new(m0002000_example_data_migration::Migration), + Box::new(MigrationWithData::new( + m0002000_example_data_migration::Migration, + )), ] } } diff --git a/migration/src/m0002000_example_data_migration.rs b/migration/src/m0002000_example_data_migration.rs index c61759b7b..edc159da7 100644 --- a/migration/src/m0002000_example_data_migration.rs +++ b/migration/src/m0002000_example_data_migration.rs @@ -1,52 +1,40 @@ use crate::{ - data::{DocumentProcessor, Sbom}, + data::{MigrationTraitWithData, Sbom, SchemaDataManager}, sbom, sea_orm::{ActiveModelTrait, IntoActiveModel, Set}, }; use sea_orm_migration::prelude::*; -use trustify_module_storage::service::{dispatch::DispatchBackend, fs::FileSystemBackend}; #[derive(DeriveMigrationName)] pub struct Migration; #[async_trait::async_trait] -impl MigrationTrait for Migration { - async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - // TODO: make this configurable - let (storage, _tmp) = FileSystemBackend::for_test() - .await - .map_err(|err| DbErr::Migration(format!("failed to create storage backend: {err}")))?; - let storage = DispatchBackend::Filesystem(storage); - - // process data - +impl MigrationTraitWithData for Migration { + async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { manager - .process( - &storage, - sbom!(async |sbom, model, tx| { - let mut model = model.into_active_model(); - match sbom { - Sbom::CycloneDx(_sbom) => { - // TODO: just an example - model.authors = Set(vec![]); - } - Sbom::Spdx(_sbom) => { - // TODO: just an example - model.authors = Set(vec![]); - } + .process(sbom!(async |sbom, model, tx| { + let mut model = model.into_active_model(); + match sbom { + Sbom::CycloneDx(_sbom) => { + // TODO: just an example + model.authors = Set(vec![]); + } + Sbom::Spdx(_sbom) => { + // TODO: just an example + model.authors = Set(vec![]); } + } - model.save(tx).await?; + model.save(tx).await?; - Ok(()) - }), - ) + Ok(()) + })) .await?; Ok(()) } - async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + async fn down(&self, _manager: &SchemaDataManager) -> Result<(), DbErr> { Ok(()) } } diff --git a/trustd/src/db.rs b/trustd/src/db.rs index af9831859..610dd4927 100644 --- a/trustd/src/db.rs +++ b/trustd/src/db.rs @@ -1,11 +1,6 @@ use postgresql_embedded::{PostgreSQL, VersionReq}; -use std::collections::HashMap; -use std::env; -use std::fs::create_dir_all; -use std::process::ExitCode; -use std::time::Duration; -use trustify_common::config::Database; -use trustify_common::db; +use std::{collections::HashMap, env, fs::create_dir_all, process::ExitCode, time::Duration}; +use trustify_common::{config::Database, db}; use trustify_infrastructure::otel::{Tracing, init_tracing}; #[derive(clap::Args, Debug)] @@ -40,6 +35,7 @@ impl Run { Err(e) => Err(e), } } + async fn refresh(self) -> anyhow::Result { match db::Database::new(&self.database).await { Ok(db) => { @@ -49,6 +45,7 @@ impl Run { Err(e) => Err(e), } } + async fn migrate(self) -> anyhow::Result { match db::Database::new(&self.database).await { Ok(db) => { From d68a2d2421893d0b7751386c3a031952aa5edd65 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 10 Sep 2025 15:21:06 +0200 Subject: [PATCH 06/47] chore: make concurrency configurable --- migration/src/data/migration.rs | 4 +++- migration/src/data/mod.rs | 31 +++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index 0262d15dd..75ec6bf6e 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -69,7 +69,9 @@ impl<'c> SchemaDataManager<'c> { where D: Document, { - self.manager.process(self.storage, f).await + self.manager + .process(self.storage, Default::default(), f) + .await } } diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 27da13ad6..b87b0d34c 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -1,5 +1,4 @@ mod migration; - pub use migration::*; use anyhow::{anyhow, bail}; @@ -12,6 +11,7 @@ use sea_orm::{ ConnectionTrait, DatabaseTransaction, DbErr, EntityTrait, ModelTrait, TransactionTrait, }; use sea_orm_migration::SchemaManager; +use std::num::NonZeroUsize; use trustify_common::id::Id; use trustify_entity::{sbom, source_document}; use trustify_module_storage::service::{StorageBackend, StorageKey, dispatch::DispatchBackend}; @@ -90,10 +90,24 @@ where ) -> anyhow::Result<()>; } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Options { + pub concurrent: NonZeroUsize, +} + +impl Default for Options { + fn default() -> Self { + Self { + concurrent: unsafe { NonZeroUsize::new_unchecked(5) }, + } + } +} + pub trait DocumentProcessor { async fn process( &self, storage: &DispatchBackend, + options: Options, f: impl Handler, ) -> anyhow::Result<(), DbErr> where @@ -101,17 +115,18 @@ pub trait DocumentProcessor { } impl<'c> DocumentProcessor for SchemaManager<'c> { - async fn process(&self, storage: &DispatchBackend, f: impl Handler) -> Result<(), DbErr> + async fn process( + &self, + storage: &DispatchBackend, + options: Options, + f: impl Handler, + ) -> Result<(), DbErr> where D: Document, { let db = self.get_connection(); let tx = db.begin().await?; - // TODO: soft-lock database - // In order to prevent new documents with an old version to be created in the meantime, we - // should soft-lock the database. - let all = D::all(&tx).await?; stream::iter(all) @@ -125,11 +140,11 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { Ok::<_, DbErr>(()) }) - .buffer_unordered(10) // TODO: make this configurable + .buffer_unordered(options.concurrent.into()) .try_collect::>() .await?; - // TODO: soft-unlock database + tx.commit().await?; Ok(()) } From b951eb4f1f9914381e10eb9265a5956567519235 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Fri, 12 Sep 2025 13:39:50 +0200 Subject: [PATCH 07/47] feat: add a way to run data migrations individually --- Cargo.lock | 2 + migration/Cargo.toml | 2 + migration/src/bin/data.rs | 154 ++++++++++++++++++++++++++++++++ migration/src/data/migration.rs | 75 +++++++++------- migration/src/data/mod.rs | 16 ++-- migration/src/lib.rs | 130 +++++++++++++++++++-------- 6 files changed, 301 insertions(+), 78 deletions(-) create mode 100644 migration/src/bin/data.rs diff --git a/Cargo.lock b/Cargo.lock index 2735ce1a1..270b0c8e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8413,6 +8413,8 @@ dependencies = [ "test-log", "tokio", "tokio-util", + "tracing", + "tracing-subscriber", "trustify-common", "trustify-db", "trustify-entity", diff --git a/migration/Cargo.toml b/migration/Cargo.toml index e54ec5ff8..c321fb034 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -24,6 +24,8 @@ serde-cyclonedx = { workspace = true } serde_json = { workspace = true } spdx-rs = { workspace = true } tokio = { workspace = true, features = ["full"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } uuid = { workspace = true, features = ["v5"] } [dev-dependencies] diff --git a/migration/src/bin/data.rs b/migration/src/bin/data.rs new file mode 100644 index 000000000..ff0dec78d --- /dev/null +++ b/migration/src/bin/data.rs @@ -0,0 +1,154 @@ +use anyhow::bail; +use clap::Parser; +use migration::{Migrator, Options, SchemaDataManager}; +use sea_orm::{ConnectOptions, Database}; +use sea_orm_migration::{IntoSchemaManagerConnection, SchemaManager}; +use std::collections::HashMap; +use trustify_module_storage::config::StorageConfig; + +#[derive(clap::Parser, Debug, Clone)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[allow(clippy::large_enum_variant)] +#[derive(clap::Subcommand, Debug, Clone)] +enum Command { + /// List all data migrations + List, + /// Run a list of migrations + Run(Run), +} + +#[derive(clap::Args, Debug, Clone)] +struct Run { + /// Migration direction to run + #[arg( + long, + value_enum, + default_value_t = Direction::Up, + overrides_with = "down" + )] + direction: Direction, + + /// Shortcut for `--direction down` + #[arg(long, action = clap::ArgAction::SetTrue, overrides_with = "direction")] + down: bool, + + // from sea_orm + #[arg( + global = true, + short = 's', + long, + env = "DATABASE_SCHEMA", + long_help = "Database schema\n \ + - For MySQL and SQLite, this argument is ignored.\n \ + - For PostgreSQL, this argument is optional with default value 'public'.\n" + )] + database_schema: Option, + + // from sea_orm + #[arg( + global = true, + short = 'u', + long, + env = "DATABASE_URL", + help = "Database URL" + )] + database_url: Option, + + #[arg()] + migrations: Vec, + + #[command(flatten)] + options: Options, + + #[command(flatten)] + storage: StorageConfig, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] +pub enum Direction { + #[default] + Up, + Down, +} + +impl Run { + fn direction(&self) -> Direction { + if self.down { + Direction::Down + } else { + self.direction + } + } + + pub async fn run(self) -> anyhow::Result<()> { + let direction = self.direction(); + + let migrations = Migrator::data_migrations() + .into_iter() + .map(|migration| (migration.name().to_string(), migration)) + .collect::>(); + + let mut running = vec![]; + + for migration in self.migrations { + let Some(migration) = migrations.get(&migration) else { + bail!("Migration {migration} not found"); + }; + running.push(migration); + } + + let storage = self.storage.into_storage(false).await?; + + let url = self + .database_url + .expect("Environment variable 'DATABASE_URL' not set"); + let schema = self.database_schema.unwrap_or_else(|| "public".to_owned()); + + let connect_options = ConnectOptions::new(url) + .set_schema_search_path(schema) + .to_owned(); + + let db = Database::connect(connect_options).await?; + + let manager = SchemaManager::new(db.into_schema_manager_connection()); + let manager = SchemaDataManager::new(&manager, &storage, &self.options); + + for run in running { + tracing::info!("Running data migration: {}", run.name()); + + match direction { + Direction::Up => run.up(&manager).await?, + Direction::Down => run.down(&manager).await?, + } + } + + Ok(()) + } +} + +impl Command { + pub async fn run(self) -> anyhow::Result<()> { + match self { + Command::Run(run) => run.run().await, + Command::List => { + for m in Migrator::data_migrations() { + println!("{}", m.name()); + } + Ok(()) + } + } + } +} + +#[tokio::main] +async fn main() { + let cli = Cli::parse(); + + tracing_subscriber::fmt::init(); + + cli.command.run().await.unwrap(); +} diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index 75ec6bf6e..806085fce 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -1,6 +1,6 @@ use crate::{ async_trait, - data::{Document, DocumentProcessor, Handler}, + data::{Document, DocumentProcessor, Handler, Options}, }; use clap::Parser; use sea_orm::DbErr; @@ -8,15 +8,14 @@ use sea_orm_migration::{MigrationName, MigrationTrait, SchemaManager}; use std::{ffi::OsString, sync::LazyLock}; use trustify_module_storage::{config::StorageConfig, service::dispatch::DispatchBackend}; -pub struct MigrationWithData -where - M: MigrationTraitWithData, -{ +pub struct MigrationWithData { pub storage: DispatchBackend, - pub migration: M, + pub options: Options, + pub migration: Box, } static STORAGE: LazyLock = LazyLock::new(init_storage); +static OPTIONS: LazyLock = LazyLock::new(init_options); #[allow(clippy::expect_used)] fn init_storage() -> DispatchBackend { @@ -33,76 +32,84 @@ fn init_storage() -> DispatchBackend { }) } -impl MigrationWithData -where - M: MigrationTraitWithData, -{ +fn init_options() -> Options { + // create from env-vars only + Options::parse_from::<_, OsString>(vec![]) +} + +impl MigrationWithData { #[allow(clippy::expect_used)] - pub fn new(migration: M) -> Self { + pub fn new(migration: Box) -> Self { Self { storage: STORAGE.clone(), + options: OPTIONS.clone(), migration, } } } -impl From for MigrationWithData +impl From for MigrationWithData where - M: MigrationTraitWithData, + M: MigrationTraitWithData + 'static, { fn from(value: M) -> Self { - MigrationWithData::new(value) + MigrationWithData::new(Box::new(value)) } } pub struct SchemaDataManager<'c> { pub manager: &'c SchemaManager<'c>, storage: &'c DispatchBackend, + options: &'c Options, } impl<'c> SchemaDataManager<'c> { - pub fn new(manager: &'c SchemaManager<'c>, storage: &'c DispatchBackend) -> Self { - Self { manager, storage } + pub fn new( + manager: &'c SchemaManager<'c>, + storage: &'c DispatchBackend, + options: &'c Options, + ) -> Self { + Self { + manager, + storage, + options, + } } pub async fn process(&self, f: impl Handler) -> Result<(), DbErr> where D: Document, { - self.manager - .process(self.storage, Default::default(), f) - .await + self.manager.process(self.storage, self.options, f).await } } #[async_trait::async_trait] -pub trait MigrationTraitWithData { +pub trait MigrationTraitWithData: MigrationName + Send + Sync { async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr>; async fn down(&self, manager: &SchemaDataManager) -> Result<(), DbErr>; } #[async_trait::async_trait] -impl MigrationTrait for MigrationWithData -where - M: MigrationTraitWithData + MigrationName + Send + Sync, -{ +impl MigrationTrait for MigrationWithData { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - self.migration - .up(&SchemaDataManager::new(manager, &self.storage)) - .await + MigrationTraitWithData::up( + &*self.migration, + &SchemaDataManager::new(manager, &self.storage, &self.options), + ) + .await } async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - self.migration - .down(&SchemaDataManager::new(manager, &self.storage)) - .await + MigrationTraitWithData::down( + &*self.migration, + &SchemaDataManager::new(manager, &self.storage, &self.options), + ) + .await } } -impl MigrationName for MigrationWithData -where - M: MigrationTraitWithData + MigrationName + Send + Sync, -{ +impl MigrationName for MigrationWithData { fn name(&self) -> &str { self.migration.name() } diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index b87b0d34c..8e7985b4c 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -90,8 +90,9 @@ where ) -> anyhow::Result<()>; } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, clap::Parser)] pub struct Options { + #[arg(long, env = "MIGRATION_DATA_CONCURRENT", default_value = "5")] pub concurrent: NonZeroUsize, } @@ -107,7 +108,7 @@ pub trait DocumentProcessor { async fn process( &self, storage: &DispatchBackend, - options: Options, + options: &Options, f: impl Handler, ) -> anyhow::Result<(), DbErr> where @@ -118,19 +119,22 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { async fn process( &self, storage: &DispatchBackend, - options: Options, + options: &Options, f: impl Handler, ) -> Result<(), DbErr> where D: Document, { let db = self.get_connection(); - let tx = db.begin().await?; + let tx = db.begin().await?; let all = D::all(&tx).await?; + drop(tx); stream::iter(all) .map(async |model| { + let tx = db.begin().await?; + let doc = D::source(&model, storage, &tx).await.map_err(|err| { DbErr::Migration(format!("Failed to load source document: {err}")) })?; @@ -138,14 +142,14 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { DbErr::Migration(format!("Failed to process document: {err}")) })?; + tx.commit().await?; + Ok::<_, DbErr>(()) }) .buffer_unordered(options.concurrent.into()) .try_collect::>() .await?; - tx.commit().await?; - Ok(()) } } diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 7973de697..62d0163d8 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -1,7 +1,7 @@ pub use sea_orm_migration::prelude::*; mod data; -use crate::data::MigrationWithData; +pub use crate::data::{MigrationTraitWithData, MigrationWithData, Options, SchemaDataManager}; mod m0000010_init; mod m0000020_add_sbom_group; @@ -37,48 +37,102 @@ mod m0001190_optimize_product_advisory_query; mod m0001200_source_document_fk_indexes; mod m0002000_example_data_migration; +#[derive(Default)] +pub struct Migrations { + all: Vec, +} + +impl IntoIterator for Migrations { + type Item = Migration; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.all.into_iter() + } +} + +pub enum Migration { + Normal(Box), + Data(Box), +} + +impl Migrations { + pub fn new() -> Self { + Self::default() + } + + pub fn normal(mut self, migration: impl MigrationTrait + 'static) -> Self { + self.all.push(Migration::Normal(Box::new(migration))); + self + } + + pub fn data(mut self, migration: impl MigrationTraitWithData + 'static) -> Self { + self.all.push(Migration::Data(Box::new(migration))); + self + } +} + pub struct Migrator; +impl Migrator { + fn migrations() -> Migrations { + Migrations::new() + .normal(m0000010_init::Migration) + .normal(m0000020_add_sbom_group::Migration) + .normal(m0000030_perf_adv_vuln::Migration) + .normal(m0000040_create_license_export::Migration) + .normal(m0000050_perf_adv_vuln2::Migration) + .normal(m0000060_perf_adv_vuln3::Migration) + .normal(m0000070_perf_adv_vuln4::Migration) + .normal(m0000080_get_purl_refactor::Migration) + .normal(m0000090_release_perf::Migration) + .normal(m0000100_perf_adv_vuln5::Migration) + .normal(m0000970_alter_importer_add_heartbeat::Migration) + .normal(m0000980_get_purl_fix::Migration) + .normal(m0000990_sbom_add_suppliers::Migration) + .normal(m0001000_sbom_non_null_suppliers::Migration) + .normal(m0001010_alter_mavenver_cmp::Migration) + .normal(m0001020_alter_pythonver_cmp::Migration) + .normal(m0001030_perf_adv_gin_index::Migration) + .normal(m0001040_alter_pythonver_cmp::Migration) + .normal(m0001050_foreign_key_cascade::Migration) + .normal(m0001060_advisory_vulnerability_indexes::Migration) + .normal(m0001070_vulnerability_scores::Migration) + .normal(m0001100_remove_get_purl::Migration) + .normal(m0001110_sbom_node_checksum_indexes::Migration) + .normal(m0001120_sbom_external_node_indexes::Migration) + .normal(m0001130_gover_cmp::Migration) + .normal(m0001140_expand_spdx_licenses_function::Migration) + .normal(m0001150_case_license_text_sbom_id_function::Migration) + .normal(m0001160_improve_expand_spdx_licenses_function::Migration) + .normal(m0001170_non_null_source_document_id::Migration) + .normal(m0001180_expand_spdx_licenses_with_mappings_function::Migration) + .normal(m0001190_optimize_product_advisory_query::Migration) + .normal(m0001200_source_document_fk_indexes::Migration) + .data(m0002000_example_data_migration::Migration) + } + + pub fn data_migrations() -> Vec> { + Self::migrations() + .into_iter() + .filter_map(|migration| match migration { + Migration::Normal(_) => None, + Migration::Data(migration) => Some(migration), + }) + .collect() + } +} + #[async_trait::async_trait] impl MigratorTrait for Migrator { fn migrations() -> Vec> { - vec![ - Box::new(m0000010_init::Migration), - Box::new(m0000020_add_sbom_group::Migration), - Box::new(m0000030_perf_adv_vuln::Migration), - Box::new(m0000040_create_license_export::Migration), - Box::new(m0000050_perf_adv_vuln2::Migration), - Box::new(m0000060_perf_adv_vuln3::Migration), - Box::new(m0000070_perf_adv_vuln4::Migration), - Box::new(m0000080_get_purl_refactor::Migration), - Box::new(m0000090_release_perf::Migration), - Box::new(m0000100_perf_adv_vuln5::Migration), - Box::new(m0000970_alter_importer_add_heartbeat::Migration), - Box::new(m0000980_get_purl_fix::Migration), - Box::new(m0000990_sbom_add_suppliers::Migration), - Box::new(m0001000_sbom_non_null_suppliers::Migration), - Box::new(m0001010_alter_mavenver_cmp::Migration), - Box::new(m0001020_alter_pythonver_cmp::Migration), - Box::new(m0001030_perf_adv_gin_index::Migration), - Box::new(m0001040_alter_pythonver_cmp::Migration), - Box::new(m0001050_foreign_key_cascade::Migration), - Box::new(m0001060_advisory_vulnerability_indexes::Migration), - Box::new(m0001070_vulnerability_scores::Migration), - Box::new(m0001100_remove_get_purl::Migration), - Box::new(m0001110_sbom_node_checksum_indexes::Migration), - Box::new(m0001120_sbom_external_node_indexes::Migration), - Box::new(m0001130_gover_cmp::Migration), - Box::new(m0001140_expand_spdx_licenses_function::Migration), - Box::new(m0001150_case_license_text_sbom_id_function::Migration), - Box::new(m0001160_improve_expand_spdx_licenses_function::Migration), - Box::new(m0001170_non_null_source_document_id::Migration), - Box::new(m0001180_expand_spdx_licenses_with_mappings_function::Migration), - Box::new(m0001190_optimize_product_advisory_query::Migration), - Box::new(m0001200_source_document_fk_indexes::Migration), - Box::new(MigrationWithData::new( - m0002000_example_data_migration::Migration, - )), - ] + Self::migrations() + .into_iter() + .map(|migration| match migration { + Migration::Normal(migration) => migration, + Migration::Data(migration) => Box::new(MigrationWithData::new(migration)), + }) + .collect() } } From 73685c746f93f4f427e5b6ff1be2a64b2d189fb5 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Fri, 19 Sep 2025 15:58:11 +0200 Subject: [PATCH 08/47] feat: add way to run data migrations from main binary --- Cargo.lock | 3 ++ common/db/src/lib.rs | 5 +++ migration/Cargo.toml | 1 + migration/src/bin/data.rs | 66 +++++++++------------------------ migration/src/data/migration.rs | 10 +---- migration/src/data/mod.rs | 48 ++++++++++++++++++++++-- migration/src/data/run.rs | 62 +++++++++++++++++++++++++++++++ migration/src/lib.rs | 46 ++++------------------- trustd/Cargo.toml | 2 + trustd/src/db.rs | 49 +++++++++++++++++++++++- 10 files changed, 193 insertions(+), 99 deletions(-) create mode 100644 migration/src/data/run.rs diff --git a/Cargo.lock b/Cargo.lock index 270b0c8e2..fd16a1ea3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8403,6 +8403,7 @@ dependencies = [ "anyhow", "bytes", "clap", + "futures", "futures-util", "sea-orm", "sea-orm-migration", @@ -8876,6 +8877,8 @@ dependencies = [ "trustify-common", "trustify-db", "trustify-infrastructure", + "trustify-migration", + "trustify-module-storage", "trustify-server", ] diff --git a/common/db/src/lib.rs b/common/db/src/lib.rs index c74ddb8ea..4e245e2cf 100644 --- a/common/db/src/lib.rs +++ b/common/db/src/lib.rs @@ -2,6 +2,7 @@ pub mod embedded; use anyhow::{Context, anyhow, ensure}; use migration::Migrator; +use migration::data::Runner; use postgresql_commands::{CommandBuilder, psql::PsqlBuilder}; use sea_orm::{ConnectionTrait, Statement}; use sea_orm_migration::prelude::MigratorTrait; @@ -121,4 +122,8 @@ impl<'a> Database<'a> { Ok(db) } + + pub async fn data_migrate(&self, runner: Runner) -> Result<(), anyhow::Error> { + runner.run::().await + } } diff --git a/migration/Cargo.toml b/migration/Cargo.toml index c321fb034..68049a5ff 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -17,6 +17,7 @@ trustify-module-storage = { workspace = true } anyhow = { workspace = true } bytes = { workspace = true } clap = { workspace = true, features = ["derive", "env"] } +futures = { workspace = true } futures-util = { workspace = true } sea-orm = { workspace = true } sea-orm-migration = { workspace = true, features = ["runtime-tokio-rustls", "sqlx-postgres", "with-uuid"] } diff --git a/migration/src/bin/data.rs b/migration/src/bin/data.rs index ff0dec78d..70bf2265d 100644 --- a/migration/src/bin/data.rs +++ b/migration/src/bin/data.rs @@ -1,9 +1,8 @@ -use anyhow::bail; use clap::Parser; -use migration::{Migrator, Options, SchemaDataManager}; -use sea_orm::{ConnectOptions, Database}; -use sea_orm_migration::{IntoSchemaManagerConnection, SchemaManager}; -use std::collections::HashMap; +use migration::{ + Migrator, + data::{Direction, MigratorWithData, Options, Runner}, +}; use trustify_module_storage::config::StorageConfig; #[derive(clap::Parser, Debug, Clone)] @@ -14,6 +13,7 @@ struct Cli { #[allow(clippy::large_enum_variant)] #[derive(clap::Subcommand, Debug, Clone)] +#[allow(clippy::large_enum_variant)] enum Command { /// List all data migrations List, @@ -68,13 +68,6 @@ struct Run { storage: StorageConfig, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] -pub enum Direction { - #[default] - Up, - Down, -} - impl Run { fn direction(&self) -> Direction { if self.down { @@ -84,47 +77,23 @@ impl Run { } } + #[allow(clippy::expect_used)] pub async fn run(self) -> anyhow::Result<()> { let direction = self.direction(); - - let migrations = Migrator::data_migrations() - .into_iter() - .map(|migration| (migration.name().to_string(), migration)) - .collect::>(); - - let mut running = vec![]; - - for migration in self.migrations { - let Some(migration) = migrations.get(&migration) else { - bail!("Migration {migration} not found"); - }; - running.push(migration); - } - let storage = self.storage.into_storage(false).await?; - let url = self - .database_url - .expect("Environment variable 'DATABASE_URL' not set"); - let schema = self.database_schema.unwrap_or_else(|| "public".to_owned()); - - let connect_options = ConnectOptions::new(url) - .set_schema_search_path(schema) - .to_owned(); - - let db = Database::connect(connect_options).await?; - - let manager = SchemaManager::new(db.into_schema_manager_connection()); - let manager = SchemaDataManager::new(&manager, &storage, &self.options); - - for run in running { - tracing::info!("Running data migration: {}", run.name()); - - match direction { - Direction::Up => run.up(&manager).await?, - Direction::Down => run.down(&manager).await?, - } + Runner { + direction, + storage, + migrations: self.migrations, + database_url: self + .database_url + .expect("Environment variable 'DATABASE_URL' not set"), + database_schema: self.database_schema, + options: self.options, } + .run::() + .await?; Ok(()) } @@ -144,6 +113,7 @@ impl Command { } } +#[allow(clippy::unwrap_used)] #[tokio::main] async fn main() { let cli = Cli::parse(); diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index 806085fce..084c9e12c 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -3,6 +3,7 @@ use crate::{ data::{Document, DocumentProcessor, Handler, Options}, }; use clap::Parser; +use futures::executor::block_on; use sea_orm::DbErr; use sea_orm_migration::{MigrationName, MigrationTrait, SchemaManager}; use std::{ffi::OsString, sync::LazyLock}; @@ -22,14 +23,7 @@ fn init_storage() -> DispatchBackend { // create from env-vars only let config = StorageConfig::parse_from::<_, OsString>(vec![]); - tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on(async { - config - .into_storage(false) - .await - .expect("Failed to create storage") - }) - }) + block_on(config.into_storage(false)).expect("task panicked") } fn init_options() -> Options { diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 8e7985b4c..6aae7328e 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -1,5 +1,8 @@ mod migration; +mod run; + pub use migration::*; +pub use run::*; use anyhow::{anyhow, bail}; use bytes::BytesMut; @@ -10,7 +13,7 @@ use futures_util::{ use sea_orm::{ ConnectionTrait, DatabaseTransaction, DbErr, EntityTrait, ModelTrait, TransactionTrait, }; -use sea_orm_migration::SchemaManager; +use sea_orm_migration::{MigrationTrait, SchemaManager}; use std::num::NonZeroUsize; use trustify_common::id::Id; use trustify_entity::{sbom, source_document}; @@ -105,12 +108,12 @@ impl Default for Options { } pub trait DocumentProcessor { - async fn process( + fn process( &self, storage: &DispatchBackend, options: &Options, f: impl Handler, - ) -> anyhow::Result<(), DbErr> + ) -> impl Future> where D: Document; } @@ -180,3 +183,42 @@ macro_rules! sbom { $crate::handler!(async |$doc: $crate::data::Sbom, $model, $tx| $body) }; } + +pub trait MigratorWithData { + fn data_migrations() -> Vec>; +} + +#[derive(Default)] +pub struct Migrations { + all: Vec, +} + +impl IntoIterator for Migrations { + type Item = Migration; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.all.into_iter() + } +} + +pub enum Migration { + Normal(Box), + Data(Box), +} + +impl Migrations { + pub fn new() -> Self { + Self::default() + } + + pub fn normal(mut self, migration: impl MigrationTrait + 'static) -> Self { + self.all.push(Migration::Normal(Box::new(migration))); + self + } + + pub fn data(mut self, migration: impl MigrationTraitWithData + 'static) -> Self { + self.all.push(Migration::Data(Box::new(migration))); + self + } +} diff --git a/migration/src/data/run.rs b/migration/src/data/run.rs new file mode 100644 index 000000000..729fbc9e4 --- /dev/null +++ b/migration/src/data/run.rs @@ -0,0 +1,62 @@ +use crate::data::{MigratorWithData, Options, SchemaDataManager}; +use anyhow::bail; +use sea_orm::{ConnectOptions, Database}; +use sea_orm_migration::{IntoSchemaManagerConnection, SchemaManager}; +use std::collections::HashMap; +use trustify_module_storage::service::dispatch::DispatchBackend; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] +pub enum Direction { + #[default] + Up, + Down, +} + +pub struct Runner { + pub database_url: String, + pub database_schema: Option, + pub storage: DispatchBackend, + pub direction: Direction, + pub migrations: Vec, + pub options: Options, +} + +impl Runner { + pub async fn run(self) -> anyhow::Result<()> { + let migrations = M::data_migrations() + .into_iter() + .map(|migration| (migration.name().to_string(), migration)) + .collect::>(); + + let mut running = vec![]; + + for migration in self.migrations { + let Some(migration) = migrations.get(&migration) else { + bail!("Migration '{migration}' not found"); + }; + running.push(migration); + } + + let schema = self.database_schema.unwrap_or_else(|| "public".to_owned()); + + let connect_options = ConnectOptions::new(self.database_url) + .set_schema_search_path(schema) + .to_owned(); + + let db = Database::connect(connect_options).await?; + + let manager = SchemaManager::new(db.into_schema_manager_connection()); + let manager = SchemaDataManager::new(&manager, &self.storage, &self.options); + + for run in running { + tracing::info!(name = run.name(), "Running data migration"); + + match self.direction { + Direction::Up => run.up(&manager).await?, + Direction::Down => run.down(&manager).await?, + } + } + + Ok(()) + } +} diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 62d0163d8..c65dedf01 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -1,7 +1,9 @@ +use crate::data::{ + Migration, MigrationTraitWithData, MigrationWithData, Migrations, MigratorWithData, +}; pub use sea_orm_migration::prelude::*; -mod data; -pub use crate::data::{MigrationTraitWithData, MigrationWithData, Options, SchemaDataManager}; +pub mod data; mod m0000010_init; mod m0000020_add_sbom_group; @@ -37,41 +39,6 @@ mod m0001190_optimize_product_advisory_query; mod m0001200_source_document_fk_indexes; mod m0002000_example_data_migration; -#[derive(Default)] -pub struct Migrations { - all: Vec, -} - -impl IntoIterator for Migrations { - type Item = Migration; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.all.into_iter() - } -} - -pub enum Migration { - Normal(Box), - Data(Box), -} - -impl Migrations { - pub fn new() -> Self { - Self::default() - } - - pub fn normal(mut self, migration: impl MigrationTrait + 'static) -> Self { - self.all.push(Migration::Normal(Box::new(migration))); - self - } - - pub fn data(mut self, migration: impl MigrationTraitWithData + 'static) -> Self { - self.all.push(Migration::Data(Box::new(migration))); - self - } -} - pub struct Migrator; impl Migrator { @@ -111,8 +78,10 @@ impl Migrator { .normal(m0001200_source_document_fk_indexes::Migration) .data(m0002000_example_data_migration::Migration) } +} - pub fn data_migrations() -> Vec> { +impl MigratorWithData for Migrator { + fn data_migrations() -> Vec> { Self::migrations() .into_iter() .filter_map(|migration| match migration { @@ -126,6 +95,7 @@ impl Migrator { #[async_trait::async_trait] impl MigratorTrait for Migrator { fn migrations() -> Vec> { + // Get all migrations, wrap data migrations. This will initialize the storage config. Self::migrations() .into_iter() .map(|migration| match migration { diff --git a/trustd/Cargo.toml b/trustd/Cargo.toml index 44622fc06..da6319bc5 100644 --- a/trustd/Cargo.toml +++ b/trustd/Cargo.toml @@ -13,6 +13,8 @@ path = "src/main.rs" trustify-common = { workspace = true } trustify-db = { workspace = true } trustify-infrastructure = { workspace = true } +trustify-migration = { workspace = true } +trustify-module-storage = { workspace = true } trustify-server = { workspace = true } anyhow = { workspace = true } diff --git a/trustd/src/db.rs b/trustd/src/db.rs index 610dd4927..980aeab54 100644 --- a/trustd/src/db.rs +++ b/trustd/src/db.rs @@ -1,7 +1,9 @@ +use migration::data::{Direction, Options, Runner}; use postgresql_embedded::{PostgreSQL, VersionReq}; use std::{collections::HashMap, env, fs::create_dir_all, process::ExitCode, time::Duration}; use trustify_common::{config::Database, db}; use trustify_infrastructure::otel::{Tracing, init_tracing}; +use trustify_module_storage::config::StorageConfig; #[derive(clap::Args, Debug)] pub struct Run { @@ -11,21 +13,39 @@ pub struct Run { pub(crate) database: Database, } -#[derive(clap::Subcommand, Debug)] +#[derive(clap::Subcommand, Debug, Clone)] pub enum Command { + /// Create database Create, + /// Run migrations (up) Migrate, + /// Remove all migrations and re-apply them (DANGER) Refresh, + /// Run specific data migrations + Data { + // Migrations to run + #[arg()] + name: Vec, + #[command(flatten)] + storage: StorageConfig, + #[command(flatten)] + options: Options, + }, } impl Run { pub async fn run(self) -> anyhow::Result { init_tracing("db-run", Tracing::Disabled); use Command::*; - match self.command { + match self.command.clone() { Create => self.create().await, Migrate => self.migrate().await, Refresh => self.refresh().await, + Data { + name, + storage, + options, + } => self.data(Direction::Up, name, storage, options).await, } } @@ -56,6 +76,31 @@ impl Run { } } + async fn data( + self, + direction: Direction, + migrations: Vec, + storage: StorageConfig, + options: Options, + ) -> anyhow::Result { + match db::Database::new(&self.database).await { + Ok(db) => { + trustify_db::Database(&db) + .data_migrate(Runner { + database_url: self.database.to_url(), + database_schema: None, + storage: storage.into_storage(false).await?, + direction, + migrations, + options, + }) + .await?; + Ok(ExitCode::SUCCESS) + } + Err(e) => Err(e), + } + } + pub async fn start(&mut self) -> anyhow::Result { init_tracing("db-start", Tracing::Disabled); log::warn!("Setting up managed DB; not suitable for production use!"); From 35ee0264005a00273336e4bf0f90ff6948227eda Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Mon, 22 Sep 2025 14:24:39 +0200 Subject: [PATCH 09/47] feat: allow spreading load across runners --- TODO.md | 4 ++ docs/book/modules/migration/pages/data.adoc | 54 +++++++++++++++++++++ migration/src/data/migration.rs | 9 +++- migration/src/data/mod.rs | 54 +++++++++++++-------- migration/src/data/partition.rs | 45 +++++++++++++++++ 5 files changed, 145 insertions(+), 21 deletions(-) create mode 100644 TODO.md create mode 100644 docs/book/modules/migration/pages/data.adoc create mode 100644 migration/src/data/partition.rs diff --git a/TODO.md b/TODO.md new file mode 100644 index 000000000..94ecc2b5d --- /dev/null +++ b/TODO.md @@ -0,0 +1,4 @@ +# ToDo + +* [ ] Allowing skipping data part of the migration +* [x] Allow concurrent instances (x of y) diff --git a/docs/book/modules/migration/pages/data.adoc b/docs/book/modules/migration/pages/data.adoc new file mode 100644 index 000000000..64716ce60 --- /dev/null +++ b/docs/book/modules/migration/pages/data.adoc @@ -0,0 +1,54 @@ += Data migration guide + +In some cases, it is necessary to also migrate data during an upgrade. This may require the re-ingestion of all stored +documents. + +This is the case, for example, a new field is added, which requires the extraction of information from the original +document. + +== Strategy + +The overall strategy for this is: + +* Prevent access to the in-migration database +* Create new database features (columns, …) in a compatible way (e.g. "allow nulls") +* Process re-ingestion of documents +* Modify database features to the target definition (e.g. "remove nullable") +* Switch to new software version +* Grant access to the new database structures + +This can be achieved with a read-only replica: + +* Prevent access to the in-migration database + * Create a read-only replica of the database + * Reconfigure trustify to use the read-only replica + * All mutable operations will fail, reporting `503 Service Unavailable` +* Create new database features +* Process re-ingestion of documents +* Modify database features to the target definition +* Switch to new software version +* Grant access to the new database structures + * Switch to new database version + * Drop old, read-only replica + +== Running the re-ingestion + +The re-ingestion can be run in two ways: + +* During the SeoORM migration +* Before the SeoORM migration + +The main difference is that when running during the SeoORM migration, you have less control over the process. It will +be driven by the SeoORM migration, and you have to wait until everything is finished. + +Running before the SeoORM migration, you can, for example, run multiple instances of the re-ingestion. You can also run +re-ingestion on a database copy, and then switch over to replace the database with the migrated version. + +Afterwards, you can run SeoORM migrations, which will then skip those DB modifications (as they are already applied) and +also skip the re-ingestion. + +== The lazy way + +The lazy way, which is the default, will simply perform those steps during the SeaORM migration. However, there are a +bunch of downsides. That's why it is not recommended for production setups. However, it may just work fine for small +test setup. Making the process a lot easier. diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index 084c9e12c..ec1d9fd08 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -1,6 +1,6 @@ use crate::{ async_trait, - data::{Document, DocumentProcessor, Handler, Options}, + data::{Document, DocumentProcessor, Handler, Options, Partition}, }; use clap::Parser; use futures::executor::block_on; @@ -55,6 +55,7 @@ pub struct SchemaDataManager<'c> { pub manager: &'c SchemaManager<'c>, storage: &'c DispatchBackend, options: &'c Options, + partition: Partition, } impl<'c> SchemaDataManager<'c> { @@ -63,10 +64,16 @@ impl<'c> SchemaDataManager<'c> { storage: &'c DispatchBackend, options: &'c Options, ) -> Self { + let partition = Partition { + current: options.current, + total: options.total, + }; + Self { manager, storage, options, + partition, } } diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 6aae7328e..6793f5882 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -1,7 +1,9 @@ mod migration; +mod partition; mod run; pub use migration::*; +pub use partition::*; pub use run::*; use anyhow::{anyhow, bail}; @@ -14,7 +16,7 @@ use sea_orm::{ ConnectionTrait, DatabaseTransaction, DbErr, EntityTrait, ModelTrait, TransactionTrait, }; use sea_orm_migration::{MigrationTrait, SchemaManager}; -use std::num::NonZeroUsize; +use std::num::{NonZeroU64, NonZeroUsize}; use trustify_common::id::Id; use trustify_entity::{sbom, source_document}; use trustify_module_storage::service::{StorageBackend, StorageKey, dispatch::DispatchBackend}; @@ -27,7 +29,7 @@ pub enum Sbom { #[allow(async_fn_in_trait)] pub trait Document: Sized + Send + Sync { - type Model: Send; + type Model: Partitionable + Send; async fn all(tx: &C) -> Result, DbErr> where @@ -95,14 +97,22 @@ where #[derive(Clone, Debug, PartialEq, Eq, clap::Parser)] pub struct Options { + /// Number of concurrent documents being processes #[arg(long, env = "MIGRATION_DATA_CONCURRENT", default_value = "5")] pub concurrent: NonZeroUsize, + + #[arg(long, env = "MIGRATION_DATA_CURRENT_RUNNER", default_value = "0")] + pub current: u64, + #[arg(long, env = "MIGRATION_DATA_TOTAL_RUNNER", default_value = "1")] + pub total: NonZeroU64, } impl Default for Options { fn default() -> Self { Self { concurrent: unsafe { NonZeroUsize::new_unchecked(5) }, + current: 0, + total: unsafe { NonZeroU64::new_unchecked(1) }, } } } @@ -128,30 +138,34 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { where D: Document, { + let partition = Partition::default(); let db = self.get_connection(); let tx = db.begin().await?; let all = D::all(&tx).await?; drop(tx); - stream::iter(all) - .map(async |model| { - let tx = db.begin().await?; - - let doc = D::source(&model, storage, &tx).await.map_err(|err| { - DbErr::Migration(format!("Failed to load source document: {err}")) - })?; - f.call(doc, model, &tx).await.map_err(|err| { - DbErr::Migration(format!("Failed to process document: {err}")) - })?; - - tx.commit().await?; - - Ok::<_, DbErr>(()) - }) - .buffer_unordered(options.concurrent.into()) - .try_collect::>() - .await?; + stream::iter( + all.into_iter() + .filter(|model| partition.is_selected::(&model)), + ) + .map(async |model| { + let tx = db.begin().await?; + + let doc = D::source(&model, storage, &tx).await.map_err(|err| { + DbErr::Migration(format!("Failed to load source document: {err}")) + })?; + f.call(doc, model, &tx) + .await + .map_err(|err| DbErr::Migration(format!("Failed to process document: {err}")))?; + + tx.commit().await?; + + Ok::<_, DbErr>(()) + }) + .buffer_unordered(options.concurrent.into()) + .try_collect::>() + .await?; Ok(()) } diff --git a/migration/src/data/partition.rs b/migration/src/data/partition.rs new file mode 100644 index 000000000..ec8a9f3b1 --- /dev/null +++ b/migration/src/data/partition.rs @@ -0,0 +1,45 @@ +use crate::data::Document; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::num::{NonZeroU64, NonZeroUsize}; +use trustify_entity::sbom; + +#[derive(Debug, Copy, Clone)] +pub struct Partition { + pub current: u64, + pub total: NonZeroU64, +} + +pub trait Partitionable { + fn hashed_id(&self) -> u64; +} + +impl Partitionable for sbom::Model { + fn hashed_id(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.sbom_id.hash(&mut hasher); + hasher.finish() + } +} + +impl Default for Partition { + fn default() -> Self { + Self::new_one() + } +} + +impl Partition { + pub const fn new_one() -> Self { + Self { + current: 0, + total: unsafe { NonZeroU64::new_unchecked(1) }, + } + } + + pub fn is_selected(&self, document: &D::Model) -> bool + where + D: Document, + D::Model: Partitionable, + { + document.hashed_id() % self.total == self.current + } +} From 7123ff792971839c8ea41492b59368c1d66f57cc Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 1 Oct 2025 11:27:18 +0200 Subject: [PATCH 10/47] refactor: extract method --- migration/src/data/migration.rs | 13 ++++--------- migration/src/data/mod.rs | 25 +++++++++++++++++++++++-- migration/src/data/partition.rs | 6 ++++-- migration/src/lib.rs | 8 +------- trustd/src/db.rs | 1 + 5 files changed, 33 insertions(+), 20 deletions(-) diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index ec1d9fd08..23e429ee8 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -1,6 +1,6 @@ use crate::{ async_trait, - data::{Document, DocumentProcessor, Handler, Options, Partition}, + data::{Document, DocumentProcessor, Handler, Options}, }; use clap::Parser; use futures::executor::block_on; @@ -32,7 +32,9 @@ fn init_options() -> Options { } impl MigrationWithData { - #[allow(clippy::expect_used)] + /// Wrap a data migration, turning it into a combined schema/data migration. + /// + /// **NOTE:** This may panic if the storage configuration is missing. pub fn new(migration: Box) -> Self { Self { storage: STORAGE.clone(), @@ -55,7 +57,6 @@ pub struct SchemaDataManager<'c> { pub manager: &'c SchemaManager<'c>, storage: &'c DispatchBackend, options: &'c Options, - partition: Partition, } impl<'c> SchemaDataManager<'c> { @@ -64,16 +65,10 @@ impl<'c> SchemaDataManager<'c> { storage: &'c DispatchBackend, options: &'c Options, ) -> Self { - let partition = Partition { - current: options.current, - total: options.total, - }; - Self { manager, storage, options, - partition, } } diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 6793f5882..7a9dd008d 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -117,6 +117,15 @@ impl Default for Options { } } +impl From<&Options> for Partition { + fn from(value: &Options) -> Self { + Self { + current: value.current, + total: value.total, + } + } +} + pub trait DocumentProcessor { fn process( &self, @@ -138,7 +147,7 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { where D: Document, { - let partition = Partition::default(); + let partition: Partition = options.into(); let db = self.get_connection(); let tx = db.begin().await?; @@ -147,7 +156,7 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { stream::iter( all.into_iter() - .filter(|model| partition.is_selected::(&model)), + .filter(|model| partition.is_selected::(model)), ) .map(async |model| { let tx = db.begin().await?; @@ -207,6 +216,18 @@ pub struct Migrations { all: Vec, } +impl Migrations { + /// Return only [`Migration::Data`] migrations. + pub fn only_data(self) -> Vec> { + self.into_iter() + .filter_map(|migration| match migration { + Migration::Normal(_) => None, + Migration::Data(migration) => Some(migration), + }) + .collect() + } +} + impl IntoIterator for Migrations { type Item = Migration; type IntoIter = std::vec::IntoIter; diff --git a/migration/src/data/partition.rs b/migration/src/data/partition.rs index ec8a9f3b1..d22bed1f7 100644 --- a/migration/src/data/partition.rs +++ b/migration/src/data/partition.rs @@ -1,6 +1,8 @@ use crate::data::Document; -use std::hash::{DefaultHasher, Hash, Hasher}; -use std::num::{NonZeroU64, NonZeroUsize}; +use std::{ + hash::{DefaultHasher, Hash, Hasher}, + num::NonZeroU64, +}; use trustify_entity::sbom; #[derive(Debug, Copy, Clone)] diff --git a/migration/src/lib.rs b/migration/src/lib.rs index c65dedf01..0810e9a51 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -82,13 +82,7 @@ impl Migrator { impl MigratorWithData for Migrator { fn data_migrations() -> Vec> { - Self::migrations() - .into_iter() - .filter_map(|migration| match migration { - Migration::Normal(_) => None, - Migration::Data(migration) => Some(migration), - }) - .collect() + Self::migrations().only_data() } } diff --git a/trustd/src/db.rs b/trustd/src/db.rs index 980aeab54..c90df1dc7 100644 --- a/trustd/src/db.rs +++ b/trustd/src/db.rs @@ -14,6 +14,7 @@ pub struct Run { } #[derive(clap::Subcommand, Debug, Clone)] +#[allow(clippy::large_enum_variant)] pub enum Command { /// Create database Create, From e2177e90923ad5b86b5c6db151362c1d57998d7e Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Tue, 7 Oct 2025 09:13:51 +0200 Subject: [PATCH 11/47] chore: test example --- .dockerignore | 7 + Cargo.lock | 5 + Cargo.toml | 1 + Containerfile | 22 +++ common/Cargo.toml | 3 +- common/src/advisory/cyclonedx.rs | 27 +++ common/src/advisory/mod.rs | 2 + data-migration.yaml | 80 +++++++++ entity/src/sbom.rs | 3 + migration/Cargo.toml | 2 + migration/src/data/document/mod.rs | 20 +++ migration/src/data/document/sbom.rs | 55 ++++++ migration/src/data/migration.rs | 23 ++- migration/src/data/mod.rs | 160 ++++++++---------- migration/src/data/run.rs | 11 ++ .../src/m0002000_example_data_migration.rs | 74 ++++++-- modules/graphql/src/sbom.rs | 2 + .../src/graph/sbom/clearly_defined.rs | 1 + modules/ingestor/src/graph/sbom/cyclonedx.rs | 3 +- modules/ingestor/src/graph/sbom/mod.rs | 6 +- modules/ingestor/src/graph/sbom/spdx.rs | 1 + .../src/service/sbom/clearly_defined.rs | 1 + trustd/src/db.rs | 81 ++++----- 23 files changed, 436 insertions(+), 154 deletions(-) create mode 100644 .dockerignore create mode 100644 Containerfile create mode 100644 common/src/advisory/cyclonedx.rs create mode 100644 data-migration.yaml create mode 100644 migration/src/data/document/mod.rs create mode 100644 migration/src/data/document/sbom.rs diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..2ac0bebaa --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.idea +.DS_Store +/data +.trustify +/target +/.dockerignore +/Containerfile diff --git a/Cargo.lock b/Cargo.lock index fd16a1ea3..6b2ebfaff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3848,7 +3848,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9375e112e4b463ec1b1c6c011953545c65a30164fbab5b581df32b3abf0dcb88" dependencies = [ "console", + "futures-core", "portable-atomic", + "tokio", "unicode-width", "unit-prefix", "web-time", @@ -8290,6 +8292,7 @@ dependencies = [ "sea-orm-migration", "sea-query", "serde", + "serde-cyclonedx", "serde_json", "spdx-expression", "spdx-rs", @@ -8405,6 +8408,8 @@ dependencies = [ "clap", "futures", "futures-util", + "humantime", + "indicatif", "sea-orm", "sea-orm-migration", "serde-cyclonedx", diff --git a/Cargo.toml b/Cargo.toml index 5281fe5af..359dbf27d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,7 @@ http = "1" human-date-parser = "0.3" humantime = "2" humantime-serde = "1" +indicatif = "0.18.0" itertools = "0.14" jsn = "0.14" json-merge-patch = "0.0.1" diff --git a/Containerfile b/Containerfile new file mode 100644 index 000000000..3445ea13d --- /dev/null +++ b/Containerfile @@ -0,0 +1,22 @@ +FROM registry.access.redhat.com/ubi9/ubi:latest AS builder + +RUN dnf install --setop install_weak_deps=false --nodocs -y git python gcc g++ cmake ninja-build openssl-devel xz + +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN mkdir /build + +COPY . /build + +WORKDIR /build + +RUN ls + +RUN rm rust-toolchain.toml + +RUN cargo build --release + +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest + +COPY --from=builder /build/target/release/trustd /usr/local/bin/ diff --git a/common/Cargo.toml b/common/Cargo.toml index e295d30aa..2cb41bffd 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -17,6 +17,7 @@ deepsize = { workspace = true } hex = { workspace = true } hide = { workspace = true } human-date-parser = { workspace = true } +humantime = { workspace = true } itertools = { workspace = true } lenient_semver = { workspace = true } log = { workspace = true } @@ -32,6 +33,7 @@ sea-orm = { workspace = true, features = ["sea-query-binder", "sqlx-postgres", " sea-orm-migration = { workspace = true } sea-query = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde-cyclonedx = { workspace = true } serde_json = { workspace = true } spdx-expression = { workspace = true } spdx-rs = { workspace = true } @@ -45,7 +47,6 @@ urlencoding = { workspace = true } utoipa = { workspace = true, features = ["url"] } uuid = { workspace = true, features = ["v5", "serde"] } walker-common = { workspace = true, features = ["bzip2", "liblzma", "flate2"] } -humantime = { workspace = true } [dev-dependencies] chrono = { workspace = true } diff --git a/common/src/advisory/cyclonedx.rs b/common/src/advisory/cyclonedx.rs new file mode 100644 index 000000000..b05b627af --- /dev/null +++ b/common/src/advisory/cyclonedx.rs @@ -0,0 +1,27 @@ +use serde_cyclonedx::cyclonedx::v_1_6::CycloneDx; +use std::collections::HashMap; + +/// extract CycloneDX SBOM general purpose properties +pub fn extract_properties(sbom: &CycloneDx) -> HashMap> { + sbom.properties + .iter() + .flatten() + .map(|e| (e.name.clone(), e.value.clone())) + .collect() +} + +/// extract CycloneDX SBOM general purpose properties, convert into [`serde_json::Value`] +pub fn extract_properties_json(sbom: &CycloneDx) -> serde_json::Value { + serde_json::Value::Object( + extract_properties(sbom) + .into_iter() + .map(|(k, v)| { + ( + k, + v.map(serde_json::Value::String) + .unwrap_or(serde_json::Value::Null), + ) + }) + .collect(), + ) +} diff --git a/common/src/advisory/mod.rs b/common/src/advisory/mod.rs index 6ae311f95..b5e0a5f6a 100644 --- a/common/src/advisory/mod.rs +++ b/common/src/advisory/mod.rs @@ -1,3 +1,5 @@ +pub mod cyclonedx; + use serde::{Deserialize, Serialize}; use std::collections::HashMap; use utoipa::ToSchema; diff --git a/data-migration.yaml b/data-migration.yaml new file mode 100644 index 000000000..77f610d36 --- /dev/null +++ b/data-migration.yaml @@ -0,0 +1,80 @@ +kind: Job +apiVersion: batch/v1 +metadata: + name: data-migration-test +spec: + completions: 4 + completionMode: Indexed + parallelism: 4 # same as completions + template: + spec: + restartPolicy: OnFailure + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/arch" + operator: In + values: ["amd64"] + containers: + - name: run + image: quay.io/ctrontesting/trustd:latest + imagePullPolicy: Always + command: + - /usr/local/bin/trustd + - db + - data + - m0002000_example_data_migration # name of the migration + env: + - name: MIGRATION_DATA_CONCURRENT + value: "5" # in-process parallelism + - name: MIGRATION_DATA_TOTAL_RUNNER + value: "4" # same as completions + - name: MIGRATION_DATA_CURRENT_RUNNER + valueFrom: + fieldRef: + fieldPath: metadata.annotations['batch.kubernetes.io/job-completion-index'] + + - name: TRUSTD_STORAGE_STRATEGY + value: s3 + - name: TRUSTD_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: storage-credentials + key: aws_access_key_id + - name: TRUSTD_S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: storage-credentials + key: aws_secret_access_key + - name: TRUSTD_S3_REGION + valueFrom: + configMapKeyRef: + name: aws-storage + key: region + - name: TRUSTD_S3_BUCKET + value: trustify-default + + - name: TRUSTD_DB_NAME + value: trustify_default + - name: TRUSTD_DB_USER + valueFrom: + secretKeyRef: + name: postgresql + key: username + - name: TRUSTD_DB_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: password + - name: TRUSTD_DB_HOST + valueFrom: + secretKeyRef: + name: postgresql + key: host + - name: TRUSTD_DB_PORT + value: "5432" + + - name: RUST_LOG + value: info diff --git a/entity/src/sbom.rs b/entity/src/sbom.rs index d8632bc4d..9063f733b 100644 --- a/entity/src/sbom.rs +++ b/entity/src/sbom.rs @@ -26,6 +26,9 @@ pub struct Model { graphql(derived(owned, into = "HashMap", with = "Labels::from")) )] pub labels: Labels, + + /// properties from the SBOM document + pub properties: serde_json::Value, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/migration/Cargo.toml b/migration/Cargo.toml index 68049a5ff..a8cacb34f 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -19,6 +19,7 @@ bytes = { workspace = true } clap = { workspace = true, features = ["derive", "env"] } futures = { workspace = true } futures-util = { workspace = true } +indicatif = { workspace = true, features = ["tokio", "futures"] } sea-orm = { workspace = true } sea-orm-migration = { workspace = true, features = ["runtime-tokio-rustls", "sqlx-postgres", "with-uuid"] } serde-cyclonedx = { workspace = true } @@ -28,6 +29,7 @@ tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } uuid = { workspace = true, features = ["v5"] } +humantime = { workspace = true } [dev-dependencies] trustify-common = { workspace = true } diff --git a/migration/src/data/document/mod.rs b/migration/src/data/document/mod.rs new file mode 100644 index 000000000..b72d89859 --- /dev/null +++ b/migration/src/data/document/mod.rs @@ -0,0 +1,20 @@ +mod sbom; +pub use sbom::*; + +use crate::data::Partitionable; +use sea_orm::{ConnectionTrait, DbErr}; +use trustify_module_storage::service::StorageBackend; + +#[allow(async_fn_in_trait)] +pub trait Document: Sized + Send + Sync { + type Model: Partitionable + Send; + + async fn all(tx: &C) -> Result, DbErr> + where + C: ConnectionTrait; + + async fn source(model: &Self::Model, storage: &S, tx: &C) -> Result + where + S: StorageBackend + Send + Sync, + C: ConnectionTrait; +} diff --git a/migration/src/data/document/sbom.rs b/migration/src/data/document/sbom.rs new file mode 100644 index 000000000..16de6763d --- /dev/null +++ b/migration/src/data/document/sbom.rs @@ -0,0 +1,55 @@ +use super::Document; +use anyhow::{anyhow, bail}; +use bytes::BytesMut; +use futures_util::TryStreamExt; +use sea_orm::prelude::*; +use trustify_common::id::Id; +use trustify_entity::{sbom, source_document}; +use trustify_module_storage::service::{StorageBackend, StorageKey}; + +#[allow(clippy::large_enum_variant)] +pub enum Sbom { + CycloneDx(serde_cyclonedx::cyclonedx::v_1_6::CycloneDx), + Spdx(spdx_rs::models::SPDX), +} + +impl Document for Sbom { + type Model = sbom::Model; + + async fn all(tx: &C) -> Result, DbErr> { + sbom::Entity::find().all(tx).await + } + + async fn source(model: &Self::Model, storage: &S, tx: &C) -> Result + where + S: StorageBackend + Send + Sync, + C: ConnectionTrait, + { + let source = model.find_related(source_document::Entity).one(tx).await?; + + let Some(source) = source else { + bail!("Missing source document ID for SBOM: {}", model.sbom_id); + }; + + let stream = storage + .retrieve( + StorageKey::try_from(Id::Sha256(source.sha256)) + .map_err(|err| anyhow!("Invalid ID: {err}"))?, + ) + .await + .map_err(|err| anyhow!("Failed to retrieve document: {err}"))? + .ok_or_else(|| anyhow!("Missing source document for SBOM: {}", model.sbom_id))?; + + stream + .try_collect::() + .await + .map_err(|err| anyhow!("Failed to collect bytes: {err}")) + .map(|bytes| bytes.freeze()) + .and_then(|bytes| { + serde_json::from_slice(&bytes) + .map(Sbom::Spdx) + .or_else(|_| serde_json::from_slice(&bytes).map(Sbom::CycloneDx)) + .map_err(|err| anyhow!("Failed to parse document: {err}")) + }) + } +} diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index 23e429ee8..0518c1f46 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -6,7 +6,7 @@ use clap::Parser; use futures::executor::block_on; use sea_orm::DbErr; use sea_orm_migration::{MigrationName, MigrationTrait, SchemaManager}; -use std::{ffi::OsString, sync::LazyLock}; +use std::{ffi::OsString, ops::Deref, sync::LazyLock}; use trustify_module_storage::{config::StorageConfig, service::dispatch::DispatchBackend}; pub struct MigrationWithData { @@ -59,6 +59,14 @@ pub struct SchemaDataManager<'c> { options: &'c Options, } +impl<'a> Deref for SchemaDataManager<'a> { + type Target = SchemaManager<'a>; + + fn deref(&self) -> &Self::Target { + self.manager + } +} + impl<'c> SchemaDataManager<'c> { pub fn new( manager: &'c SchemaManager<'c>, @@ -72,10 +80,21 @@ impl<'c> SchemaDataManager<'c> { } } - pub async fn process(&self, f: impl Handler) -> Result<(), DbErr> + pub async fn process(&self, name: &N, f: impl Handler) -> Result<(), DbErr> where D: Document, + N: MigrationName + Send + Sync, { + if self.options.skip_all { + // we skip all migration + return Ok(()); + } + + if self.options.skip.iter().any(|s| s == name.name()) { + // we skip a list of migrations, and it's on the list + return Ok(()); + } + self.manager.process(self.storage, self.options, f).await } } diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 7a9dd008d..85eeb99d2 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -1,86 +1,25 @@ +mod document; mod migration; mod partition; mod run; +pub use document::*; pub use migration::*; pub use partition::*; pub use run::*; -use anyhow::{anyhow, bail}; -use bytes::BytesMut; use futures_util::{ StreamExt, stream::{self, TryStreamExt}, }; -use sea_orm::{ - ConnectionTrait, DatabaseTransaction, DbErr, EntityTrait, ModelTrait, TransactionTrait, -}; +use indicatif::{ProgressBar, ProgressStyle}; +use sea_orm::{DatabaseTransaction, DbErr, TransactionTrait}; use sea_orm_migration::{MigrationTrait, SchemaManager}; -use std::num::{NonZeroU64, NonZeroUsize}; -use trustify_common::id::Id; -use trustify_entity::{sbom, source_document}; -use trustify_module_storage::service::{StorageBackend, StorageKey, dispatch::DispatchBackend}; - -#[allow(clippy::large_enum_variant)] -pub enum Sbom { - CycloneDx(serde_cyclonedx::cyclonedx::v_1_6::CycloneDx), - Spdx(spdx_rs::models::SPDX), -} - -#[allow(async_fn_in_trait)] -pub trait Document: Sized + Send + Sync { - type Model: Partitionable + Send; - - async fn all(tx: &C) -> Result, DbErr> - where - C: ConnectionTrait; - - async fn source(model: &Self::Model, storage: &S, tx: &C) -> Result - where - S: StorageBackend + Send + Sync, - C: ConnectionTrait; -} - -impl Document for Sbom { - type Model = sbom::Model; - - async fn all(tx: &C) -> Result, DbErr> { - sbom::Entity::find().all(tx).await - } - - async fn source(model: &Self::Model, storage: &S, tx: &C) -> Result - where - S: StorageBackend + Send + Sync, - C: ConnectionTrait, - { - let source = model.find_related(source_document::Entity).one(tx).await?; - - let Some(source) = source else { - bail!("Missing source document ID for SBOM: {}", model.sbom_id); - }; - - let stream = storage - .retrieve( - StorageKey::try_from(Id::Sha256(source.sha256)) - .map_err(|err| anyhow!("Invalid ID: {err}"))?, - ) - .await - .map_err(|err| anyhow!("Failed to retrieve document: {err}"))? - .ok_or_else(|| anyhow!("Missing source document for SBOM: {}", model.sbom_id))?; - - stream - .try_collect::() - .await - .map_err(|err| anyhow!("Failed to collect bytes: {err}")) - .map(|bytes| bytes.freeze()) - .and_then(|bytes| { - serde_json::from_slice(&bytes) - .map(Sbom::Spdx) - .or_else(|_| serde_json::from_slice(&bytes).map(Sbom::CycloneDx)) - .map_err(|err| anyhow!("Failed to parse document: {err}")) - }) - } -} +use std::{ + num::{NonZeroU64, NonZeroUsize}, + sync::Arc, +}; +use trustify_module_storage::service::dispatch::DispatchBackend; #[allow(async_fn_in_trait)] pub trait Handler: Send @@ -101,10 +40,25 @@ pub struct Options { #[arg(long, env = "MIGRATION_DATA_CONCURRENT", default_value = "5")] pub concurrent: NonZeroUsize, + /// The instance number of the current runner (zero based) #[arg(long, env = "MIGRATION_DATA_CURRENT_RUNNER", default_value = "0")] pub current: u64, + /// The total number of runners #[arg(long, env = "MIGRATION_DATA_TOTAL_RUNNER", default_value = "1")] pub total: NonZeroU64, + + /// Skip running all data migrations + #[arg( + long, + env = "MIGRATION_DATA_SKIP_ALL", + default_value_t, + conflicts_with = "skip" + )] + pub skip_all: bool, + + /// Skip the provided list of data migrations + #[arg(long, env = "MIGRATION_DATA_SKIP", conflicts_with = "skip_all")] + pub skip: Vec, } impl Default for Options { @@ -113,6 +67,8 @@ impl Default for Options { concurrent: unsafe { NonZeroUsize::new_unchecked(5) }, current: 0, total: unsafe { NonZeroU64::new_unchecked(1) }, + skip_all: false, + skip: vec![], } } } @@ -151,30 +107,50 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { let db = self.get_connection(); let tx = db.begin().await?; - let all = D::all(&tx).await?; + let all: Vec<_> = D::all(&tx) + .await? + .into_iter() + .filter(|model| partition.is_selected::(model)) + .collect(); drop(tx); - stream::iter( - all.into_iter() - .filter(|model| partition.is_selected::(model)), - ) - .map(async |model| { - let tx = db.begin().await?; - - let doc = D::source(&model, storage, &tx).await.map_err(|err| { - DbErr::Migration(format!("Failed to load source document: {err}")) - })?; - f.call(doc, model, &tx) - .await - .map_err(|err| DbErr::Migration(format!("Failed to process document: {err}")))?; - - tx.commit().await?; - - Ok::<_, DbErr>(()) - }) - .buffer_unordered(options.concurrent.into()) - .try_collect::>() - .await?; + let pb = Arc::new(ProgressBar::new(all.len() as u64)); + pb.set_style( + ProgressStyle::with_template( + "{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len} ({eta})", + ) + .map_err(|err| DbErr::Migration(err.to_string()))? + .progress_chars("##-"), + ); + + let pb = Some(pb); + + stream::iter(all) + .map(async |model| { + let tx = db.begin().await?; + + let doc = D::source(&model, storage, &tx).await.map_err(|err| { + DbErr::Migration(format!("Failed to load source document: {err}")) + })?; + f.call(doc, model, &tx).await.map_err(|err| { + DbErr::Migration(format!("Failed to process document: {err}")) + })?; + + tx.commit().await?; + + if let Some(pb) = &pb { + pb.inc(1); + } + + Ok::<_, DbErr>(()) + }) + .buffer_unordered(options.concurrent.into()) + .try_collect::>() + .await?; + + if let Some(pb) = &pb { + pb.finish_with_message("Done"); + } Ok(()) } diff --git a/migration/src/data/run.rs b/migration/src/data/run.rs index 729fbc9e4..e73e55e01 100644 --- a/migration/src/data/run.rs +++ b/migration/src/data/run.rs @@ -3,6 +3,7 @@ use anyhow::bail; use sea_orm::{ConnectOptions, Database}; use sea_orm_migration::{IntoSchemaManagerConnection, SchemaManager}; use std::collections::HashMap; +use std::time::SystemTime; use trustify_module_storage::service::dispatch::DispatchBackend; #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] @@ -51,10 +52,20 @@ impl Runner { for run in running { tracing::info!(name = run.name(), "Running data migration"); + let start = SystemTime::now(); + match self.direction { Direction::Up => run.up(&manager).await?, Direction::Down => run.down(&manager).await?, } + + if let Ok(duration) = start.elapsed() { + tracing::info!( + name = run.name(), + "Took {}", + humantime::Duration::from(duration) + ) + } } Ok(()) diff --git a/migration/src/m0002000_example_data_migration.rs b/migration/src/m0002000_example_data_migration.rs index edc159da7..970327f59 100644 --- a/migration/src/m0002000_example_data_migration.rs +++ b/migration/src/m0002000_example_data_migration.rs @@ -1,9 +1,10 @@ use crate::{ - data::{MigrationTraitWithData, Sbom, SchemaDataManager}, + data::{MigrationTraitWithData, Sbom as SbomDoc, SchemaDataManager}, sbom, - sea_orm::{ActiveModelTrait, IntoActiveModel, Set}, }; +use sea_orm::{ActiveModelTrait, IntoActiveModel, Set}; use sea_orm_migration::prelude::*; +use trustify_common::advisory::cyclonedx::extract_properties_json; #[derive(DeriveMigrationName)] pub struct Migration; @@ -12,29 +13,68 @@ pub struct Migration; impl MigrationTraitWithData for Migration { async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { manager - .process(sbom!(async |sbom, model, tx| { - let mut model = model.into_active_model(); - match sbom { - Sbom::CycloneDx(_sbom) => { - // TODO: just an example - model.authors = Set(vec![]); - } - Sbom::Spdx(_sbom) => { - // TODO: just an example - model.authors = Set(vec![]); + .alter_table( + Table::alter() + .table(Sbom::Table) + .add_column_if_not_exists( + ColumnDef::new(Sbom::Properties) + .json() + .default(serde_json::Value::Null) + .to_owned(), + ) + .to_owned(), + ) + .await?; + + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .modify_column(ColumnDef::new(Sbom::Properties).not_null().to_owned()) + .to_owned(), + ) + .await?; + + manager + .process( + self, + sbom!(async |sbom, model, tx| { + let mut model = model.into_active_model(); + match sbom { + SbomDoc::CycloneDx(sbom) => { + model.properties = Set(extract_properties_json(&sbom)); + } + SbomDoc::Spdx(_sbom) => { + model.properties = Set(serde_json::Value::Object(Default::default())); + } } - } - model.save(tx).await?; + model.save(tx).await?; - Ok(()) - })) + Ok(()) + }), + ) .await?; Ok(()) } - async fn down(&self, _manager: &SchemaDataManager) -> Result<(), DbErr> { + async fn down(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .drop_column(Sbom::Properties) + .to_owned(), + ) + .await?; + Ok(()) } } + +#[derive(DeriveIden)] +enum Sbom { + Table, + Properties, +} diff --git a/modules/graphql/src/sbom.rs b/modules/graphql/src/sbom.rs index 39f590448..f931aecb3 100644 --- a/modules/graphql/src/sbom.rs +++ b/modules/graphql/src/sbom.rs @@ -26,6 +26,7 @@ impl SbomQuery { suppliers: sbom_context.sbom.suppliers, source_document_id: sbom_context.sbom.source_document_id, data_licenses: sbom_context.sbom.data_licenses, + properties: sbom_context.sbom.properties, }), Ok(None) => Err(FieldError::new("SBOM not found")), Err(err) => Err(FieldError::from(err)), @@ -68,6 +69,7 @@ impl SbomQuery { suppliers: sbom.sbom.suppliers, source_document_id: sbom.sbom.source_document_id, data_licenses: sbom.sbom.data_licenses, + properties: sbom.sbom.properties, }) }) .collect() diff --git a/modules/ingestor/src/graph/sbom/clearly_defined.rs b/modules/ingestor/src/graph/sbom/clearly_defined.rs index 55ed37190..5dc388b69 100644 --- a/modules/ingestor/src/graph/sbom/clearly_defined.rs +++ b/modules/ingestor/src/graph/sbom/clearly_defined.rs @@ -89,6 +89,7 @@ impl Into for &Curation { authors: vec!["ClearlyDefined: Community-Curated".to_string()], suppliers: vec![], data_licenses: vec![], + properties: Default::default(), } } } diff --git a/modules/ingestor/src/graph/sbom/cyclonedx.rs b/modules/ingestor/src/graph/sbom/cyclonedx.rs index 31f0e5954..00e3ce351 100644 --- a/modules/ingestor/src/graph/sbom/cyclonedx.rs +++ b/modules/ingestor/src/graph/sbom/cyclonedx.rs @@ -27,7 +27,7 @@ use serde_cyclonedx::cyclonedx::v_1_6::{ use std::{borrow::Cow, str::FromStr}; use time::{OffsetDateTime, format_description::well_known::Iso8601}; use tracing::instrument; -use trustify_common::{cpe::Cpe, purl::Purl}; +use trustify_common::{advisory::cyclonedx::extract_properties_json, cpe::Cpe, purl::Purl}; use trustify_entity::relationship::Relationship; use uuid::Uuid; @@ -128,6 +128,7 @@ impl<'a> From> for SbomInformation { authors, suppliers, data_licenses, + properties: extract_properties_json(sbom), } } } diff --git a/modules/ingestor/src/graph/sbom/mod.rs b/modules/ingestor/src/graph/sbom/mod.rs index dcc3fca1e..be2c4f174 100644 --- a/modules/ingestor/src/graph/sbom/mod.rs +++ b/modules/ingestor/src/graph/sbom/mod.rs @@ -6,7 +6,6 @@ pub mod processor; pub mod spdx; mod common; - pub use common::*; use super::error::Error; @@ -50,6 +49,8 @@ pub struct SbomInformation { pub suppliers: Vec, /// The licenses of the data itself, if known. pub data_licenses: Vec, + /// general purpose properties from the SBOM + pub properties: serde_json::Value, } impl From<()> for SbomInformation { @@ -110,6 +111,7 @@ impl Graph { authors, suppliers, data_licenses, + properties, } = info.into(); let new_id = match self @@ -137,6 +139,8 @@ impl Graph { source_document_id: Set(new_id), labels: Set(labels.into().validate()?), data_licenses: Set(data_licenses), + + properties: Set(properties), }; let node_model = sbom_node::ActiveModel { diff --git a/modules/ingestor/src/graph/sbom/spdx.rs b/modules/ingestor/src/graph/sbom/spdx.rs index 106e84251..0f0129209 100644 --- a/modules/ingestor/src/graph/sbom/spdx.rs +++ b/modules/ingestor/src/graph/sbom/spdx.rs @@ -96,6 +96,7 @@ impl<'a> From> for SbomInformation { .clone(), suppliers: suppliers(sbom), data_licenses: vec![value.0.document_creation_information.data_license.clone()], + properties: Default::default(), } } } diff --git a/modules/ingestor/src/service/sbom/clearly_defined.rs b/modules/ingestor/src/service/sbom/clearly_defined.rs index 1e66083cc..5acb55151 100644 --- a/modules/ingestor/src/service/sbom/clearly_defined.rs +++ b/modules/ingestor/src/service/sbom/clearly_defined.rs @@ -66,6 +66,7 @@ impl<'g> ClearlyDefinedLoader<'g> { authors: vec!["ClearlyDefined Definitions".to_string()], suppliers: vec![], data_licenses: vec![], + properties: Default::default(), }, &tx, ) diff --git a/trustd/src/db.rs b/trustd/src/db.rs index c90df1dc7..bb0d61da2 100644 --- a/trustd/src/db.rs +++ b/trustd/src/db.rs @@ -23,30 +23,18 @@ pub enum Command { /// Remove all migrations and re-apply them (DANGER) Refresh, /// Run specific data migrations - Data { - // Migrations to run - #[arg()] - name: Vec, - #[command(flatten)] - storage: StorageConfig, - #[command(flatten)] - options: Options, - }, + Data(Data), } impl Run { pub async fn run(self) -> anyhow::Result { init_tracing("db-run", Tracing::Disabled); use Command::*; - match self.command.clone() { + match self.command { Create => self.create().await, Migrate => self.migrate().await, Refresh => self.refresh().await, - Data { - name, - storage, - options, - } => self.data(Direction::Up, name, storage, options).await, + Data(data) => data.run(Direction::Up, self.database).await, } } @@ -77,31 +65,6 @@ impl Run { } } - async fn data( - self, - direction: Direction, - migrations: Vec, - storage: StorageConfig, - options: Options, - ) -> anyhow::Result { - match db::Database::new(&self.database).await { - Ok(db) => { - trustify_db::Database(&db) - .data_migrate(Runner { - database_url: self.database.to_url(), - database_schema: None, - storage: storage.into_storage(false).await?, - direction, - migrations, - options, - }) - .await?; - Ok(ExitCode::SUCCESS) - } - Err(e) => Err(e), - } - } - pub async fn start(&mut self) -> anyhow::Result { init_tracing("db-start", Tracing::Disabled); log::warn!("Setting up managed DB; not suitable for production use!"); @@ -147,3 +110,41 @@ impl Run { Ok(postgresql) } } + +#[derive(clap::Args, Debug, Clone)] +pub struct Data { + /// Migrations to run + #[arg()] + name: Vec, + #[command(flatten)] + storage: StorageConfig, + #[command(flatten)] + options: Options, +} + +impl Data { + pub async fn run(self, direction: Direction, database: Database) -> anyhow::Result { + let Self { + name: migrations, + storage, + options, + } = self; + + match db::Database::new(&database).await { + Ok(db) => { + trustify_db::Database(&db) + .data_migrate(Runner { + database_url: database.to_url(), + database_schema: None, + storage: storage.into_storage(false).await?, + direction, + migrations, + options, + }) + .await?; + Ok(ExitCode::SUCCESS) + } + Err(e) => Err(e), + } + } +} From 078d7dd7163c42774da274e13abea8ac200b3364 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Thu, 9 Oct 2025 15:37:08 +0200 Subject: [PATCH 12/47] chore: show number of processed documents --- migration/src/data/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 85eeb99d2..aa3034b1b 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -114,7 +114,8 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { .collect(); drop(tx); - let pb = Arc::new(ProgressBar::new(all.len() as u64)); + let count = all.len(); + let pb = Arc::new(ProgressBar::new(count as u64)); pb.set_style( ProgressStyle::with_template( "{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len} ({eta})", @@ -152,6 +153,8 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { pb.finish_with_message("Done"); } + tracing::info!("Processed {count} documents"); + Ok(()) } } From 8c85241cad8e25c4b7dcdffbb200da2cc34b5603 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Fri, 10 Oct 2025 09:59:46 +0200 Subject: [PATCH 13/47] chore: add advisory to example --- Cargo.lock | 3 + migration/Cargo.toml | 5 +- migration/src/data/document/advisory.rs | 39 +++++++++ migration/src/data/document/mod.rs | 44 +++++++++- migration/src/data/document/sbom.rs | 46 ++++------ migration/src/data/migration.rs | 1 + migration/src/data/mod.rs | 14 ++++ migration/src/data/partition.rs | 15 +++- migration/src/lib.rs | 6 +- ...> m0002000_example_sbom_data_migration.rs} | 3 + ...0002010_example_advisory_data_migration.rs | 83 +++++++++++++++++++ 11 files changed, 221 insertions(+), 38 deletions(-) create mode 100644 migration/src/data/document/advisory.rs rename migration/src/{m0002000_example_data_migration.rs => m0002000_example_sbom_data_migration.rs} (94%) create mode 100644 migration/src/m0002010_example_advisory_data_migration.rs diff --git a/Cargo.lock b/Cargo.lock index 6b2ebfaff..3fc863f5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8406,10 +8406,13 @@ dependencies = [ "anyhow", "bytes", "clap", + "csaf", + "cve", "futures", "futures-util", "humantime", "indicatif", + "osv", "sea-orm", "sea-orm-migration", "serde-cyclonedx", diff --git a/migration/Cargo.toml b/migration/Cargo.toml index a8cacb34f..1ef7fa876 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -17,9 +17,13 @@ trustify-module-storage = { workspace = true } anyhow = { workspace = true } bytes = { workspace = true } clap = { workspace = true, features = ["derive", "env"] } +csaf = { workspace = true } +cve = { workspace = true } futures = { workspace = true } futures-util = { workspace = true } +humantime = { workspace = true } indicatif = { workspace = true, features = ["tokio", "futures"] } +osv = { workspace = true, features = ["schema"] } sea-orm = { workspace = true } sea-orm-migration = { workspace = true, features = ["runtime-tokio-rustls", "sqlx-postgres", "with-uuid"] } serde-cyclonedx = { workspace = true } @@ -29,7 +33,6 @@ tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } uuid = { workspace = true, features = ["v5"] } -humantime = { workspace = true } [dev-dependencies] trustify-common = { workspace = true } diff --git a/migration/src/data/document/advisory.rs b/migration/src/data/document/advisory.rs new file mode 100644 index 000000000..c5fc3bf2d --- /dev/null +++ b/migration/src/data/document/advisory.rs @@ -0,0 +1,39 @@ +use super::Document; +use bytes::Bytes; +use sea_orm::prelude::*; +use trustify_entity::advisory; +use trustify_module_storage::service::StorageBackend; + +#[allow(clippy::large_enum_variant)] +pub enum Advisory { + Cve(cve::Cve), + Csaf(csaf::Csaf), + Osv(osv::schema::Vulnerability), + Other(Bytes), +} + +impl From for Advisory { + fn from(value: Bytes) -> Self { + serde_json::from_slice(&value) + .map(Advisory::Cve) + .or_else(|_| serde_json::from_slice(&value).map(Advisory::Csaf)) + .or_else(|_| serde_json::from_slice(&value).map(Advisory::Osv)) + .unwrap_or_else(|_err| Advisory::Other(value)) + } +} + +impl Document for Advisory { + type Model = advisory::Model; + + async fn all(tx: &C) -> Result, DbErr> { + advisory::Entity::find().all(tx).await + } + + async fn source(model: &Self::Model, storage: &S, tx: &C) -> Result + where + S: StorageBackend + Send + Sync, + C: ConnectionTrait, + { + super::load(model.source_document_id, storage, tx).await + } +} diff --git a/migration/src/data/document/mod.rs b/migration/src/data/document/mod.rs index b72d89859..de3491077 100644 --- a/migration/src/data/document/mod.rs +++ b/migration/src/data/document/mod.rs @@ -1,9 +1,18 @@ +mod advisory; + +pub use advisory::*; +use anyhow::{anyhow, bail}; +use bytes::{Bytes, BytesMut}; +use futures_util::TryStreamExt; mod sbom; pub use sbom::*; use crate::data::Partitionable; -use sea_orm::{ConnectionTrait, DbErr}; -use trustify_module_storage::service::StorageBackend; +use sea_orm::{ConnectionTrait, DbErr, EntityTrait}; +use trustify_common::id::Id; +use trustify_entity::source_document; +use trustify_module_storage::service::{StorageBackend, StorageKey}; +use uuid::Uuid; #[allow(async_fn_in_trait)] pub trait Document: Sized + Send + Sync { @@ -18,3 +27,34 @@ pub trait Document: Sized + Send + Sync { S: StorageBackend + Send + Sync, C: ConnectionTrait; } + +pub(crate) async fn load( + id: Uuid, + storage: &(impl StorageBackend + Send + Sync), + tx: &impl ConnectionTrait, +) -> anyhow::Result +where + D: Document + From, +{ + let source = source_document::Entity::find_by_id(id).one(tx).await?; + + let Some(source) = source else { + bail!("Missing source document entry for: {id}"); + }; + + let stream = storage + .retrieve( + StorageKey::try_from(Id::Sha256(source.sha256)) + .map_err(|err| anyhow!("Invalid ID: {err}"))?, + ) + .await + .map_err(|err| anyhow!("Failed to retrieve document: {err}"))? + .ok_or_else(|| anyhow!("Missing source document for: {id}"))?; + + stream + .try_collect::() + .await + .map_err(|err| anyhow!("Failed to collect bytes: {err}")) + .map(|bytes| bytes.freeze()) + .map(|bytes| bytes.into()) +} diff --git a/migration/src/data/document/sbom.rs b/migration/src/data/document/sbom.rs index 16de6763d..cf964938f 100644 --- a/migration/src/data/document/sbom.rs +++ b/migration/src/data/document/sbom.rs @@ -1,16 +1,23 @@ use super::Document; -use anyhow::{anyhow, bail}; -use bytes::BytesMut; -use futures_util::TryStreamExt; +use bytes::Bytes; use sea_orm::prelude::*; -use trustify_common::id::Id; -use trustify_entity::{sbom, source_document}; -use trustify_module_storage::service::{StorageBackend, StorageKey}; +use trustify_entity::sbom; +use trustify_module_storage::service::StorageBackend; #[allow(clippy::large_enum_variant)] pub enum Sbom { CycloneDx(serde_cyclonedx::cyclonedx::v_1_6::CycloneDx), Spdx(spdx_rs::models::SPDX), + Other(Bytes), +} + +impl From for Sbom { + fn from(value: Bytes) -> Self { + serde_json::from_slice(&value) + .map(Sbom::Spdx) + .or_else(|_| serde_json::from_slice(&value).map(Sbom::CycloneDx)) + .unwrap_or_else(|_err| Sbom::Other(value)) + } } impl Document for Sbom { @@ -25,31 +32,6 @@ impl Document for Sbom { S: StorageBackend + Send + Sync, C: ConnectionTrait, { - let source = model.find_related(source_document::Entity).one(tx).await?; - - let Some(source) = source else { - bail!("Missing source document ID for SBOM: {}", model.sbom_id); - }; - - let stream = storage - .retrieve( - StorageKey::try_from(Id::Sha256(source.sha256)) - .map_err(|err| anyhow!("Invalid ID: {err}"))?, - ) - .await - .map_err(|err| anyhow!("Failed to retrieve document: {err}"))? - .ok_or_else(|| anyhow!("Missing source document for SBOM: {}", model.sbom_id))?; - - stream - .try_collect::() - .await - .map_err(|err| anyhow!("Failed to collect bytes: {err}")) - .map(|bytes| bytes.freeze()) - .and_then(|bytes| { - serde_json::from_slice(&bytes) - .map(Sbom::Spdx) - .or_else(|_| serde_json::from_slice(&bytes).map(Sbom::CycloneDx)) - .map_err(|err| anyhow!("Failed to parse document: {err}")) - }) + super::load(model.source_document_id, storage, tx).await } } diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index 0518c1f46..8722a0138 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -80,6 +80,7 @@ impl<'c> SchemaDataManager<'c> { } } + /// Run a data migration pub async fn process(&self, name: &N, f: impl Handler) -> Result<(), DbErr> where D: Document, diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index aa3034b1b..091d792cd 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -159,6 +159,7 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { } } +/// A handler for data migration of documents. #[macro_export] macro_rules! handler { (async | $doc:ident: $doc_ty:ty, $model:ident, $tx:ident | $body:block) => {{ @@ -179,6 +180,9 @@ macro_rules! handler { }}; } +/// A handler for SBOMs. +/// +/// See: [`handler!`]. #[macro_export] macro_rules! sbom { (async | $doc:ident, $model:ident, $tx:ident | $body:block) => { @@ -186,6 +190,16 @@ macro_rules! sbom { }; } +/// A handler for advisories. +/// +/// See: [`handler!`]. +#[macro_export] +macro_rules! advisories { + (async | $doc:ident, $model:ident, $tx:ident | $body:block) => { + $crate::handler!(async |$doc: $crate::data::Advisory, $model, $tx| $body) + }; +} + pub trait MigratorWithData { fn data_migrations() -> Vec>; } diff --git a/migration/src/data/partition.rs b/migration/src/data/partition.rs index d22bed1f7..57d7fb9f0 100644 --- a/migration/src/data/partition.rs +++ b/migration/src/data/partition.rs @@ -3,7 +3,7 @@ use std::{ hash::{DefaultHasher, Hash, Hasher}, num::NonZeroU64, }; -use trustify_entity::sbom; +use trustify_entity::{advisory, sbom}; #[derive(Debug, Copy, Clone)] pub struct Partition { @@ -11,7 +11,12 @@ pub struct Partition { pub total: NonZeroU64, } +/// A thing which can be distributed over different partitions via a hashed id. +/// +/// The idea is that the thing returns a hash ID, which can then be distributed over partitions +/// by using a "X of Y" approach. Where the thing is processed when "ID modulo Y == X". pub trait Partitionable { + /// Get the hashed ID for the thing. fn hashed_id(&self) -> u64; } @@ -23,6 +28,14 @@ impl Partitionable for sbom::Model { } } +impl Partitionable for advisory::Model { + fn hashed_id(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.id.hash(&mut hasher); + hasher.finish() + } +} + impl Default for Partition { fn default() -> Self { Self::new_one() diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 0810e9a51..664a2a1ea 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -37,7 +37,8 @@ mod m0001170_non_null_source_document_id; mod m0001180_expand_spdx_licenses_with_mappings_function; mod m0001190_optimize_product_advisory_query; mod m0001200_source_document_fk_indexes; -mod m0002000_example_data_migration; +mod m0002000_example_sbom_data_migration; +mod m0002010_example_advisory_data_migration; pub struct Migrator; @@ -76,7 +77,8 @@ impl Migrator { .normal(m0001180_expand_spdx_licenses_with_mappings_function::Migration) .normal(m0001190_optimize_product_advisory_query::Migration) .normal(m0001200_source_document_fk_indexes::Migration) - .data(m0002000_example_data_migration::Migration) + .data(m0002000_example_sbom_data_migration::Migration) + .data(m0002010_example_advisory_data_migration::Migration) } } diff --git a/migration/src/m0002000_example_data_migration.rs b/migration/src/m0002000_example_sbom_data_migration.rs similarity index 94% rename from migration/src/m0002000_example_data_migration.rs rename to migration/src/m0002000_example_sbom_data_migration.rs index 970327f59..603b24031 100644 --- a/migration/src/m0002000_example_data_migration.rs +++ b/migration/src/m0002000_example_sbom_data_migration.rs @@ -47,6 +47,9 @@ impl MigrationTraitWithData for Migration { SbomDoc::Spdx(_sbom) => { model.properties = Set(serde_json::Value::Object(Default::default())); } + SbomDoc::Other(_) => { + // we ignore others + } } model.save(tx).await?; diff --git a/migration/src/m0002010_example_advisory_data_migration.rs b/migration/src/m0002010_example_advisory_data_migration.rs new file mode 100644 index 000000000..603b24031 --- /dev/null +++ b/migration/src/m0002010_example_advisory_data_migration.rs @@ -0,0 +1,83 @@ +use crate::{ + data::{MigrationTraitWithData, Sbom as SbomDoc, SchemaDataManager}, + sbom, +}; +use sea_orm::{ActiveModelTrait, IntoActiveModel, Set}; +use sea_orm_migration::prelude::*; +use trustify_common::advisory::cyclonedx::extract_properties_json; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTraitWithData for Migration { + async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .add_column_if_not_exists( + ColumnDef::new(Sbom::Properties) + .json() + .default(serde_json::Value::Null) + .to_owned(), + ) + .to_owned(), + ) + .await?; + + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .modify_column(ColumnDef::new(Sbom::Properties).not_null().to_owned()) + .to_owned(), + ) + .await?; + + manager + .process( + self, + sbom!(async |sbom, model, tx| { + let mut model = model.into_active_model(); + match sbom { + SbomDoc::CycloneDx(sbom) => { + model.properties = Set(extract_properties_json(&sbom)); + } + SbomDoc::Spdx(_sbom) => { + model.properties = Set(serde_json::Value::Object(Default::default())); + } + SbomDoc::Other(_) => { + // we ignore others + } + } + + model.save(tx).await?; + + Ok(()) + }), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .drop_column(Sbom::Properties) + .to_owned(), + ) + .await?; + + Ok(()) + } +} + +#[derive(DeriveIden)] +enum Sbom { + Table, + Properties, +} From 0bcac1efa1046b04906dc9e6635288e43f38e301 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Fri, 10 Oct 2025 15:59:48 +0200 Subject: [PATCH 14/47] feat: ingest scores --- Cargo.lock | 2 + entity/src/advisory_vulnerability.rs | 3 + entity/src/advisory_vulnerability_score.rs | 127 ++++++++++++++ entity/src/lib.rs | 1 + migration/Cargo.toml | 2 + migration/src/data/mod.rs | 2 +- ...0002010_example_advisory_data_migration.rs | 155 ++++++++++++++---- modules/ingestor/src/graph/cvss.rs | 57 +++++++ modules/ingestor/src/graph/mod.rs | 1 + .../src/service/advisory/cve/loader.rs | 6 + .../ingestor/src/service/advisory/cve/mod.rs | 24 +++ .../src/service/advisory/osv/loader.rs | 15 +- .../ingestor/src/service/advisory/osv/mod.rs | 76 ++++++++- 13 files changed, 433 insertions(+), 38 deletions(-) create mode 100644 entity/src/advisory_vulnerability_score.rs create mode 100644 modules/ingestor/src/graph/cvss.rs diff --git a/Cargo.lock b/Cargo.lock index 3fc863f5d..6cecb0f04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8418,6 +8418,7 @@ dependencies = [ "serde-cyclonedx", "serde_json", "spdx-rs", + "strum 0.27.2", "test-context", "test-log", "tokio", @@ -8427,6 +8428,7 @@ dependencies = [ "trustify-common", "trustify-db", "trustify-entity", + "trustify-module-ingestor", "trustify-module-storage", "trustify-test-context", "uuid", diff --git a/entity/src/advisory_vulnerability.rs b/entity/src/advisory_vulnerability.rs index d21d80284..fd2687e02 100644 --- a/entity/src/advisory_vulnerability.rs +++ b/entity/src/advisory_vulnerability.rs @@ -37,6 +37,9 @@ pub enum Relation { #[sea_orm(has_many = "super::purl_status::Entity")] PurlStatus, + + #[sea_orm(has_many = "super::advisory_vulnerability_score::Entity")] + Score, } impl Related for Entity { diff --git a/entity/src/advisory_vulnerability_score.rs b/entity/src/advisory_vulnerability_score.rs new file mode 100644 index 000000000..6b78e56bc --- /dev/null +++ b/entity/src/advisory_vulnerability_score.rs @@ -0,0 +1,127 @@ +use crate::{advisory, advisory_vulnerability, cvss3, vulnerability}; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "advisory_vulnerability_score")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: Uuid, + + pub advisory_id: Uuid, + pub vulnerability_id: String, + + pub r#type: ScoreType, + pub vector: String, + pub score: f64, + pub severity: Severity, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::advisory_vulnerability::Entity", + from = "(super::advisory_vulnerability_score::Column::AdvisoryId, super::advisory_vulnerability_score::Column::VulnerabilityId)" + to = "(super::advisory_vulnerability::Column::AdvisoryId, super::advisory_vulnerability::Column::VulnerabilityId)" + )] + AdvisoryVulnerability, + #[sea_orm( + belongs_to = "super::advisory::Entity", + from = "super::advisory_vulnerability_score::Column::AdvisoryId" + to = "super::advisory::Column::Id" + )] + Advisory, + #[sea_orm( + belongs_to = "super::vulnerability::Entity", + from = "super::advisory_vulnerability_score::Column::VulnerabilityId" + to = "super::vulnerability::Column::Id" + )] + Vulnerability, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Advisory.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Vulnerability.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::AdvisoryVulnerability.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} + +// score type + +#[derive(Debug, Copy, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "score_type")] +pub enum ScoreType { + #[sea_orm(string_value = "2.0")] + V2_0, + #[sea_orm(string_value = "3.0")] + V3_0, + #[sea_orm(string_value = "3.1")] + V3_1, + #[sea_orm(string_value = "4.0")] + V4_0, +} + +// severity + +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + EnumIter, + DeriveActiveEnum, + strum::EnumString, + strum::Display, + strum::VariantNames, +)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "severity")] +#[strum(serialize_all = "lowercase")] +pub enum Severity { + #[sea_orm(string_value = "none")] + None, + #[sea_orm(string_value = "low")] + Low, + #[sea_orm(string_value = "medium")] + Medium, + #[sea_orm(string_value = "high")] + High, + #[sea_orm(string_value = "critical")] + Critical, +} + +impl From for Severity { + fn from(value: cvss3::Severity) -> Self { + match value { + cvss3::Severity::None => Self::None, + cvss3::Severity::Low => Self::Low, + cvss3::Severity::Medium => Self::Medium, + cvss3::Severity::High => Self::High, + cvss3::Severity::Critical => Self::Critical, + } + } +} + +impl From for cvss3::Severity { + fn from(value: Severity) -> Self { + match value { + Severity::None => Self::None, + Severity::Low => Self::Low, + Severity::Medium => Self::Medium, + Severity::High => Self::High, + Severity::Critical => Self::Critical, + } + } +} diff --git a/entity/src/lib.rs b/entity/src/lib.rs index e39362eef..58450aa43 100644 --- a/entity/src/lib.rs +++ b/entity/src/lib.rs @@ -1,5 +1,6 @@ pub mod advisory; pub mod advisory_vulnerability; +pub mod advisory_vulnerability_score; pub mod base_purl; pub mod cpe; pub mod cvss3; diff --git a/migration/Cargo.toml b/migration/Cargo.toml index 1ef7fa876..c5af6efc5 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -12,6 +12,7 @@ path = "src/lib.rs" [dependencies] trustify-common = { workspace = true } trustify-entity = { workspace = true } +trustify-module-ingestor = { workspace = true } trustify-module-storage = { workspace = true } anyhow = { workspace = true } @@ -29,6 +30,7 @@ sea-orm-migration = { workspace = true, features = ["runtime-tokio-rustls", "sql serde-cyclonedx = { workspace = true } serde_json = { workspace = true } spdx-rs = { workspace = true } +strum = { workspace = true, features = ["derive"] } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 091d792cd..60f543e9e 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -194,7 +194,7 @@ macro_rules! sbom { /// /// See: [`handler!`]. #[macro_export] -macro_rules! advisories { +macro_rules! advisory { (async | $doc:ident, $model:ident, $tx:ident | $body:block) => { $crate::handler!(async |$doc: $crate::data::Advisory, $model, $tx| $body) }; diff --git a/migration/src/m0002010_example_advisory_data_migration.rs b/migration/src/m0002010_example_advisory_data_migration.rs index 603b24031..cb5f5042d 100644 --- a/migration/src/m0002010_example_advisory_data_migration.rs +++ b/migration/src/m0002010_example_advisory_data_migration.rs @@ -1,10 +1,12 @@ use crate::{ - data::{MigrationTraitWithData, Sbom as SbomDoc, SchemaDataManager}, - sbom, + advisory, + data::{Advisory as AdvisoryDoc, MigrationTraitWithData, SchemaDataManager}, }; -use sea_orm::{ActiveModelTrait, IntoActiveModel, Set}; +use sea_orm::{EntityTrait, IntoActiveModel, sea_query::extension::postgres::Type}; use sea_orm_migration::prelude::*; -use trustify_common::advisory::cyclonedx::extract_properties_json; +use strum::VariantNames; +use trustify_entity::{advisory, advisory_vulnerability_score}; +use trustify_module_ingestor::{graph::cvss::ScoreCreator, service::advisory::osv::extract_scores}; #[derive(DeriveMigrationName)] pub struct Migration; @@ -13,24 +15,71 @@ pub struct Migration; impl MigrationTraitWithData for Migration { async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { manager - .alter_table( - Table::alter() - .table(Sbom::Table) - .add_column_if_not_exists( - ColumnDef::new(Sbom::Properties) - .json() - .default(serde_json::Value::Null) - .to_owned(), - ) + .create_type( + Type::create() + .as_enum(Severity::Table) + .values(Severity::VARIANTS.iter().skip(1).copied()) .to_owned(), ) .await?; manager - .alter_table( - Table::alter() - .table(Sbom::Table) - .modify_column(ColumnDef::new(Sbom::Properties).not_null().to_owned()) + .create_type( + Type::create() + .as_enum(ScoreType::Table) + .values(ScoreType::VARIANTS.iter().skip(1).copied()) + .to_owned(), + ) + .await?; + + manager + .create_table( + Table::create() + .table(AdvisoryVulnerabilityScore::Table) + .if_not_exists() + .col( + ColumnDef::new(AdvisoryVulnerabilityScore::Id) + .uuid() + .not_null() + .primary_key() + .to_owned(), + ) + .col( + ColumnDef::new(AdvisoryVulnerabilityScore::AdvisoryId) + .uuid() + .not_null() + .to_owned(), + ) + .col( + ColumnDef::new(AdvisoryVulnerabilityScore::VulnerabilityId) + .uuid() + .not_null() + .to_owned(), + ) + .col( + ColumnDef::new(AdvisoryVulnerabilityScore::Type) + .custom(ScoreType::Table) + .not_null() + .to_owned(), + ) + .col( + ColumnDef::new(AdvisoryVulnerabilityScore::Vector) + .string() + .not_null() + .to_owned(), + ) + .col( + ColumnDef::new(AdvisoryVulnerabilityScore::Score) + .float() + .not_null() + .to_owned(), + ) + .col( + ColumnDef::new(AdvisoryVulnerabilityScore::Severity) + .custom(Severity::Table) + .not_null() + .to_owned(), + ) .to_owned(), ) .await?; @@ -38,22 +87,20 @@ impl MigrationTraitWithData for Migration { manager .process( self, - sbom!(async |sbom, model, tx| { - let mut model = model.into_active_model(); - match sbom { - SbomDoc::CycloneDx(sbom) => { - model.properties = Set(extract_properties_json(&sbom)); + advisory!(async |advisory, model, tx| { + match advisory { + AdvisoryDoc::Cve(advisory) => {} + AdvisoryDoc::Csaf(advisory) => {} + AdvisoryDoc::Osv(advisory) => { + let mut creator = ScoreCreator::new(model.id); + extract_scores(&advisory, &mut creator); + creator.create(tx).await?; } - SbomDoc::Spdx(_sbom) => { - model.properties = Set(serde_json::Value::Object(Default::default())); - } - SbomDoc::Other(_) => { + _ => { // we ignore others } } - model.save(tx).await?; - Ok(()) }), ) @@ -64,20 +111,60 @@ impl MigrationTraitWithData for Migration { async fn down(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { manager - .alter_table( - Table::alter() - .table(Sbom::Table) - .drop_column(Sbom::Properties) + .drop_table( + Table::drop() + .table(AdvisoryVulnerabilityScore::Table) + .if_exists() .to_owned(), ) .await?; + manager + .drop_type(Type::drop().if_exists().name("severity").to_owned()) + .await?; + + manager + .drop_type(Type::drop().if_exists().name("score_type").to_owned()) + .await?; + Ok(()) } } #[derive(DeriveIden)] -enum Sbom { +enum AdvisoryVulnerabilityScore { + Table, + Id, + AdvisoryId, + VulnerabilityId, + Type, + Vector, + Score, + Severity, +} + +#[derive(DeriveIden, strum::VariantNames, strum::Display)] +#[allow(unused)] +enum ScoreType { + Table, + #[strum(to_string = "2.0")] + V2_0, + #[strum(to_string = "3.0")] + V3_0, + #[strum(to_string = "3.1")] + V3_1, + #[strum(to_string = "4.0")] + V4_0, +} + +#[derive(DeriveIden, strum::VariantNames, strum::Display)] +#[strum(serialize_all = "lowercase")] +#[allow(unused)] +enum Severity { Table, - Properties, + None, + Low, + Medium, + High, + Critical, } diff --git a/modules/ingestor/src/graph/cvss.rs b/modules/ingestor/src/graph/cvss.rs new file mode 100644 index 000000000..51772f40e --- /dev/null +++ b/modules/ingestor/src/graph/cvss.rs @@ -0,0 +1,57 @@ +use sea_orm::{ColumnTrait, ConnectionTrait, DbErr, EntityTrait, QueryFilter, Set}; +use trustify_common::db::chunk::EntityChunkedIter; +use trustify_entity::advisory_vulnerability_score; +use uuid::Uuid; + +#[derive(Debug)] +pub struct ScoreCreator { + advisory_id: Uuid, + scores: Vec, +} + +impl ScoreCreator { + pub fn new(advisory_id: Uuid) -> Self { + Self { + advisory_id, + scores: Vec::new(), + } + } + + pub fn add(&mut self, model: impl Into) { + self.scores.push(model.into()); + } + + pub fn extend( + &mut self, + items: impl IntoIterator, + ) { + self.scores.extend(items) + } + + pub async fn create(mut self, db: &impl ConnectionTrait) -> Result<(), DbErr> { + // delete existing entries + + advisory_vulnerability_score::Entity::delete_many() + .filter(advisory_vulnerability_score::Column::AdvisoryId.eq(self.advisory_id)) + .exec(db) + .await?; + + // set advisory + + for score in &mut self.scores { + score.advisory_id = Set(self.advisory_id); + } + + // insert chunked + + for batch in &self.scores.chunked() { + advisory_vulnerability_score::Entity::insert_many(batch) + .exec(db) + .await?; + } + + // done + + Ok(()) + } +} diff --git a/modules/ingestor/src/graph/mod.rs b/modules/ingestor/src/graph/mod.rs index 3379e8fca..88fdc20ac 100644 --- a/modules/ingestor/src/graph/mod.rs +++ b/modules/ingestor/src/graph/mod.rs @@ -1,5 +1,6 @@ pub mod advisory; pub mod cpe; +pub mod cvss; pub mod db_context; pub mod error; pub mod organization; diff --git a/modules/ingestor/src/service/advisory/cve/loader.rs b/modules/ingestor/src/service/advisory/cve/loader.rs index 351e7bf34..bc9ea17d4 100644 --- a/modules/ingestor/src/service/advisory/cve/loader.rs +++ b/modules/ingestor/src/service/advisory/cve/loader.rs @@ -1,3 +1,5 @@ +use crate::graph::cvss::ScoreCreator; +use crate::service::advisory::cve::extract_scores; use crate::{ graph::{ Graph, @@ -94,6 +96,10 @@ impl<'g> CveLoader<'g> { .ingest_advisory(id, labels, digests, advisory_info, &tx) .await?; + let mut score_creator = ScoreCreator::new(advisory.advisory.id); + extract_scores(&cve, &mut score_creator); + score_creator.create(&tx).await?; + // Link the advisory to the backing vulnerability let advisory_vuln = advisory .link_to_vulnerability( diff --git a/modules/ingestor/src/service/advisory/cve/mod.rs b/modules/ingestor/src/service/advisory/cve/mod.rs index 9ec27d9bd..5ff23ffc4 100644 --- a/modules/ingestor/src/service/advisory/cve/mod.rs +++ b/modules/ingestor/src/service/advisory/cve/mod.rs @@ -1,2 +1,26 @@ +use crate::graph::cvss::ScoreCreator; +use cve::Cve; + pub mod divination; pub mod loader; + +pub fn extract_scores(cve: &Cve, score_creator: &mut ScoreCreator) { + let Cve::Published(cve) = cve else { + return; + }; + + for metrics in &cve.containers.cna.metrics { + if let Some(value) = &metrics.cvss_v2_0 { + // TODO: add score to creator + } + if let Some(value) = &metrics.cvss_v3_0 { + // TODO: add score to creator + } + if let Some(value) = &metrics.cvss_v3_1 { + // TODO: add score to creator + } + if let Some(value) = &metrics.cvss_v4_0 { + // TODO: add score to creator + } + } +} diff --git a/modules/ingestor/src/service/advisory/osv/loader.rs b/modules/ingestor/src/service/advisory/osv/loader.rs index 301e85f9d..7396ca481 100644 --- a/modules/ingestor/src/service/advisory/osv/loader.rs +++ b/modules/ingestor/src/service/advisory/osv/loader.rs @@ -1,3 +1,4 @@ +use crate::service::advisory::osv::extract_vulnerability_ids; use crate::{ graph::{ Graph, @@ -6,6 +7,7 @@ use crate::{ advisory_vulnerability::AdvisoryVulnerabilityContext, version::{Version, VersionInfo, VersionSpec}, }, + cvss::ScoreCreator, purl::{ self, creator::PurlCreator, @@ -16,7 +18,7 @@ use crate::{ model::IngestResult, service::{ Error, Warnings, - advisory::osv::{prefix::get_well_known_prefixes, translate}, + advisory::osv::{extract_scores, prefix::get_well_known_prefixes, translate}, }, }; use osv::schema::{Ecosystem, Event, Range, RangeType, ReferenceType, SeverityType, Vulnerability}; @@ -94,10 +96,20 @@ impl<'g> OsvLoader<'g> { vuln_creator.create(&tx).await?; let mut purl_creator = PurlCreator::new(); +<<<<<<< HEAD let mut purl_status_creator = PurlStatusCreator::new(); let mut base_purls = HashSet::new(); for cve_id in &cve_ids { +======= + let mut score_creator = ScoreCreator::new(advisory.advisory.id); + + extract_scores(&osv, &mut score_creator); + + for cve_id in extract_vulnerability_ids(&osv) { + self.graph.ingest_vulnerability(&cve_id, (), &tx).await?; + +>>>>>>> 26d7d87d (feat: ingest scores) let advisory_vuln = advisory .link_to_vulnerability( cve_id, @@ -310,6 +322,7 @@ impl<'g> OsvLoader<'g> { } purl_creator.create(&tx).await?; + score_creator.create(&tx).await?; // Create base PURLs for range-based status entries purl::batch_create_base_purls(base_purls, &tx).await?; diff --git a/modules/ingestor/src/service/advisory/osv/mod.rs b/modules/ingestor/src/service/advisory/osv/mod.rs index 90cfb623d..d776c802f 100644 --- a/modules/ingestor/src/service/advisory/osv/mod.rs +++ b/modules/ingestor/src/service/advisory/osv/mod.rs @@ -3,8 +3,11 @@ mod prefix; pub mod loader; pub mod translate; -use crate::service::Error; -use osv::schema::Vulnerability; +use crate::{graph::cvss::ScoreCreator, service::Error}; +use osv::schema::{SeverityType, Vulnerability}; +use sea_orm::{NotSet, Set}; +use trustify_entity::advisory_vulnerability_score::{self, ScoreType}; +use uuid::Uuid; /// Load a [`Vulnerability`] from YAML, using the "classic" enum representation. pub fn from_yaml(data: &[u8]) -> Result { @@ -34,3 +37,72 @@ pub fn parse(buffer: &[u8]) -> Result { Ok(osv) } + +/// extract vulnerability IDs +pub fn extract_vulnerability_ids(osv: &Vulnerability) -> impl IntoIterator { + osv.aliases + .iter() + .flat_map(|aliases| aliases.iter().filter(|e| e.starts_with("CVE-"))) + .map(|s| s.as_str()) +} + +/// extract scores from OSV +pub fn extract_scores(osv: &Vulnerability, creator: &mut ScoreCreator) { + // TODO: validate score type by prefix + let scores = osv + .severity + .iter() + .flatten() + .flat_map(|severity| match severity.severity_type { + SeverityType::CVSSv2 => Some(( + ScoreType::V2_0, + severity.score.clone(), + 10f64, // TODO: replace with actual evaluated score + advisory_vulnerability_score::Severity::Critical, // TODO: replace with actual evaluated severity + )), + SeverityType::CVSSv3 => Some(( + match severity.score.starts_with("CVSS:3.1/") { + true => ScoreType::V3_1, + false => ScoreType::V3_0, + }, + severity.score.clone(), + 10f64, // TODO: replace with actual evaluated score + advisory_vulnerability_score::Severity::Critical, // TODO: replace with actual evaluated severity + )), + SeverityType::CVSSv4 => Some(( + ScoreType::V4_0, + severity.score.clone(), + 10f64, // TODO: replace with actual evaluated score + advisory_vulnerability_score::Severity::Critical, // TODO: replace with actual evaluated severity + )), + + _ => None, + }) + .map( + move |(r#type, vector, score, severity)| advisory_vulnerability_score::ActiveModel { + id: Set(Uuid::now_v7()), + r#type: Set(r#type), + vector: Set(vector), + score: Set(score), + severity: Set(severity), + advisory_id: NotSet, + vulnerability_id: NotSet, + }, + ); + + // get all vulnerability IDs + + let ids = extract_vulnerability_ids(osv) + .into_iter() + .collect::>(); + + // create scores for each vulnerability (alias) + + creator.extend(scores.into_iter().flat_map(|score| { + ids.iter().map(move |id| { + let mut score = score.clone(); + score.vulnerability_id = Set(id.to_string()); + score + }) + })); +} From 72ee72561c1ef20c5696fe935b88309fa7ee2f6c Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Tue, 14 Oct 2025 10:17:52 +0200 Subject: [PATCH 15/47] chore: continue work on PoC --- ...0002010_example_advisory_data_migration.rs | 16 +++++++++---- modules/ingestor/src/graph/cvss.rs | 23 +++++++++++-------- .../src/service/advisory/cve/loader.rs | 8 ++++--- .../src/service/advisory/osv/loader.rs | 7 +----- modules/ingestor/tests/parallel.rs | 3 +-- 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/migration/src/m0002010_example_advisory_data_migration.rs b/migration/src/m0002010_example_advisory_data_migration.rs index cb5f5042d..1d722e993 100644 --- a/migration/src/m0002010_example_advisory_data_migration.rs +++ b/migration/src/m0002010_example_advisory_data_migration.rs @@ -2,11 +2,13 @@ use crate::{ advisory, data::{Advisory as AdvisoryDoc, MigrationTraitWithData, SchemaDataManager}, }; -use sea_orm::{EntityTrait, IntoActiveModel, sea_query::extension::postgres::Type}; +use sea_orm::sea_query::extension::postgres::Type; use sea_orm_migration::prelude::*; use strum::VariantNames; -use trustify_entity::{advisory, advisory_vulnerability_score}; -use trustify_module_ingestor::{graph::cvss::ScoreCreator, service::advisory::osv::extract_scores}; +use trustify_module_ingestor::{ + graph::cvss::ScoreCreator, + service::advisory::{cve, osv}, +}; #[derive(DeriveMigrationName)] pub struct Migration; @@ -89,11 +91,15 @@ impl MigrationTraitWithData for Migration { self, advisory!(async |advisory, model, tx| { match advisory { - AdvisoryDoc::Cve(advisory) => {} + AdvisoryDoc::Cve(advisory) => { + let mut creator = ScoreCreator::new(model.id); + cve::extract_scores(&advisory, &mut creator); + creator.create(tx).await?; + } AdvisoryDoc::Csaf(advisory) => {} AdvisoryDoc::Osv(advisory) => { let mut creator = ScoreCreator::new(model.id); - extract_scores(&advisory, &mut creator); + osv::extract_scores(&advisory, &mut creator); creator.create(tx).await?; } _ => { diff --git a/modules/ingestor/src/graph/cvss.rs b/modules/ingestor/src/graph/cvss.rs index 51772f40e..bff2bb1bb 100644 --- a/modules/ingestor/src/graph/cvss.rs +++ b/modules/ingestor/src/graph/cvss.rs @@ -1,5 +1,4 @@ use sea_orm::{ColumnTrait, ConnectionTrait, DbErr, EntityTrait, QueryFilter, Set}; -use trustify_common::db::chunk::EntityChunkedIter; use trustify_entity::advisory_vulnerability_score; use uuid::Uuid; @@ -28,27 +27,33 @@ impl ScoreCreator { self.scores.extend(items) } - pub async fn create(mut self, db: &impl ConnectionTrait) -> Result<(), DbErr> { + pub async fn create(self, db: &C) -> Result<(), DbErr> + where + C: ConnectionTrait, + { + let Self { + advisory_id, + mut scores, + } = self; + // delete existing entries advisory_vulnerability_score::Entity::delete_many() - .filter(advisory_vulnerability_score::Column::AdvisoryId.eq(self.advisory_id)) + .filter(advisory_vulnerability_score::Column::AdvisoryId.eq(advisory_id)) .exec(db) .await?; // set advisory - for score in &mut self.scores { + for score in &mut scores { score.advisory_id = Set(self.advisory_id); } // insert chunked - for batch in &self.scores.chunked() { - advisory_vulnerability_score::Entity::insert_many(batch) - .exec(db) - .await?; - } + advisory_vulnerability_score::Entity::insert_many(scores) + .exec(db) + .await?; // done diff --git a/modules/ingestor/src/service/advisory/cve/loader.rs b/modules/ingestor/src/service/advisory/cve/loader.rs index bc9ea17d4..22a09b442 100644 --- a/modules/ingestor/src/service/advisory/cve/loader.rs +++ b/modules/ingestor/src/service/advisory/cve/loader.rs @@ -1,5 +1,3 @@ -use crate::graph::cvss::ScoreCreator; -use crate::service::advisory::cve::extract_scores; use crate::{ graph::{ Graph, @@ -7,6 +5,7 @@ use crate::{ AdvisoryInformation, AdvisoryVulnerabilityInformation, version::{Version, VersionInfo, VersionSpec}, }, + cvss::ScoreCreator, purl::{ self, status_creator::{PurlStatusCreator, PurlStatusEntry}, @@ -14,7 +13,10 @@ use crate::{ vulnerability::{VulnerabilityInformation, creator::VulnerabilityCreator}, }, model::IngestResult, - service::{Error, Warnings, advisory::cve::divination::divine_purl}, + service::{ + Error, Warnings, + advisory::cve::{divination::divine_purl, extract_scores}, + }, }; use cve::{ Cve, Timestamp, diff --git a/modules/ingestor/src/service/advisory/osv/loader.rs b/modules/ingestor/src/service/advisory/osv/loader.rs index 7396ca481..32409a317 100644 --- a/modules/ingestor/src/service/advisory/osv/loader.rs +++ b/modules/ingestor/src/service/advisory/osv/loader.rs @@ -96,20 +96,15 @@ impl<'g> OsvLoader<'g> { vuln_creator.create(&tx).await?; let mut purl_creator = PurlCreator::new(); -<<<<<<< HEAD let mut purl_status_creator = PurlStatusCreator::new(); let mut base_purls = HashSet::new(); - - for cve_id in &cve_ids { -======= let mut score_creator = ScoreCreator::new(advisory.advisory.id); extract_scores(&osv, &mut score_creator); for cve_id in extract_vulnerability_ids(&osv) { - self.graph.ingest_vulnerability(&cve_id, (), &tx).await?; + self.graph.ingest_vulnerability(cve_id, (), &tx).await?; ->>>>>>> 26d7d87d (feat: ingest scores) let advisory_vuln = advisory .link_to_vulnerability( cve_id, diff --git a/modules/ingestor/tests/parallel.rs b/modules/ingestor/tests/parallel.rs index 172958355..15058dbf8 100644 --- a/modules/ingestor/tests/parallel.rs +++ b/modules/ingestor/tests/parallel.rs @@ -9,14 +9,13 @@ use test_context::{futures, test_context}; use test_log::test; use tracing::instrument; use trustify_common::{cpe::Cpe, purl::Purl, sbom::spdx::parse_spdx}; -use trustify_module_ingestor::service::Cache; use trustify_module_ingestor::{ graph::{ cpe::CpeCreator, purl::creator::PurlCreator, sbom::{LicenseCreator, LicenseInfo}, }, - service::{Discard, Format}, + service::{Cache, Discard, Format}, }; use trustify_test_context::{TrustifyContext, document_bytes, spdx::fix_spdx_rels}; use uuid::Uuid; From e48f9bd7d89c1e13a20c10b18521c62babdfde0f Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Tue, 14 Oct 2025 15:06:10 +0200 Subject: [PATCH 16/47] chore: add foreign key --- ...0002010_example_advisory_data_migration.rs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/migration/src/m0002010_example_advisory_data_migration.rs b/migration/src/m0002010_example_advisory_data_migration.rs index 1d722e993..dff694687 100644 --- a/migration/src/m0002010_example_advisory_data_migration.rs +++ b/migration/src/m0002010_example_advisory_data_migration.rs @@ -82,6 +82,19 @@ impl MigrationTraitWithData for Migration { .not_null() .to_owned(), ) + .foreign_key( + ForeignKey::create() + .from_col(AdvisoryVulnerabilityScore::AdvisoryId) + .from_col(AdvisoryVulnerabilityScore::VulnerabilityId) + .to( + AdvisoryVulnerability::Table, + ( + AdvisoryVulnerability::AdvisoryId, + AdvisoryVulnerability::VulnerabilityId, + ), + ) + .on_delete(ForeignKeyAction::Cascade), + ) .to_owned(), ) .await?; @@ -137,6 +150,13 @@ impl MigrationTraitWithData for Migration { } } +#[derive(DeriveIden)] +enum AdvisoryVulnerability { + Table, + AdvisoryId, + VulnerabilityId, +} + #[derive(DeriveIden)] enum AdvisoryVulnerabilityScore { Table, From 8b4bec0566b84b09ecc35293a3f1dab731d4f109 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 15 Oct 2025 09:38:45 +0200 Subject: [PATCH 17/47] chore: implement for csaf --- Cargo.lock | 2 ++ Cargo.toml | 1 + entity/Cargo.toml | 1 + entity/src/advisory_vulnerability_score.rs | 12 +++++++ ...0002010_example_advisory_data_migration.rs | 13 ++++---- modules/ingestor/Cargo.toml | 1 + modules/ingestor/src/graph/cvss.rs | 32 ++++++++++++++++++ .../src/service/advisory/csaf/loader.rs | 6 ++++ .../ingestor/src/service/advisory/csaf/mod.rs | 33 +++++++++++++++++++ .../ingestor/src/service/advisory/cve/mod.rs | 2 +- 10 files changed, 96 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cecb0f04..e995936f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8344,6 +8344,7 @@ dependencies = [ "anyhow", "async-graphql", "cpe", + "cvss", "deepsize", "log", "rstest", @@ -8654,6 +8655,7 @@ dependencies = [ "cpe", "csaf", "cve", + "cvss", "hex", "humantime", "jsn", diff --git a/Cargo.toml b/Cargo.toml index 359dbf27d..e1708d09d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ csaf = { version = "0.5.0", default-features = false } csaf-walker = { version = "0.14.1", default-features = false } csv = "1.3.0" cve = "0.5.0" +cvss = "2" deepsize = "0.2.0" fixedbitset = "0.5.7" flate2 = "1.0.35" diff --git a/entity/Cargo.toml b/entity/Cargo.toml index 36c2ce44c..f0e217657 100644 --- a/entity/Cargo.toml +++ b/entity/Cargo.toml @@ -13,6 +13,7 @@ trustify-common = { workspace = true } trustify-cvss = { workspace = true } cpe = { workspace = true } +cvss = { workspace = true } deepsize = { workspace = true } schemars = { workspace = true } sea-orm = { workspace = true, features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json", "postgres-array"] } diff --git a/entity/src/advisory_vulnerability_score.rs b/entity/src/advisory_vulnerability_score.rs index 6b78e56bc..7bb4e71c5 100644 --- a/entity/src/advisory_vulnerability_score.rs +++ b/entity/src/advisory_vulnerability_score.rs @@ -125,3 +125,15 @@ impl From for cvss3::Severity { } } } + +impl From for Severity { + fn from(value: cvss::Severity) -> Self { + match value { + cvss::Severity::None => Self::None, + cvss::Severity::Low => Self::Low, + cvss::Severity::Medium => Self::Medium, + cvss::Severity::High => Self::High, + cvss::Severity::Critical => Self::Critical, + } + } +} diff --git a/migration/src/m0002010_example_advisory_data_migration.rs b/migration/src/m0002010_example_advisory_data_migration.rs index dff694687..a732cf698 100644 --- a/migration/src/m0002010_example_advisory_data_migration.rs +++ b/migration/src/m0002010_example_advisory_data_migration.rs @@ -7,7 +7,7 @@ use sea_orm_migration::prelude::*; use strum::VariantNames; use trustify_module_ingestor::{ graph::cvss::ScoreCreator, - service::advisory::{cve, osv}, + service::advisory::{csaf, cve, osv}, }; #[derive(DeriveMigrationName)] @@ -103,23 +103,24 @@ impl MigrationTraitWithData for Migration { .process( self, advisory!(async |advisory, model, tx| { + let mut creator = ScoreCreator::new(model.id); match advisory { AdvisoryDoc::Cve(advisory) => { - let mut creator = ScoreCreator::new(model.id); cve::extract_scores(&advisory, &mut creator); - creator.create(tx).await?; } - AdvisoryDoc::Csaf(advisory) => {} + AdvisoryDoc::Csaf(advisory) => { + csaf::extract_scores(&advisory, &mut creator); + } AdvisoryDoc::Osv(advisory) => { - let mut creator = ScoreCreator::new(model.id); osv::extract_scores(&advisory, &mut creator); - creator.create(tx).await?; } _ => { // we ignore others } } + creator.create(tx).await?; + Ok(()) }), ) diff --git a/modules/ingestor/Cargo.toml b/modules/ingestor/Cargo.toml index c26d10b9d..4fe24e7f3 100644 --- a/modules/ingestor/Cargo.toml +++ b/modules/ingestor/Cargo.toml @@ -19,6 +19,7 @@ bytes = { workspace = true } cpe = { workspace = true } csaf = { workspace = true } cve = { workspace = true } +cvss = { workspace = true } hex = { workspace = true } humantime = { workspace = true } jsn = { workspace = true } diff --git a/modules/ingestor/src/graph/cvss.rs b/modules/ingestor/src/graph/cvss.rs index bff2bb1bb..ef28d54ec 100644 --- a/modules/ingestor/src/graph/cvss.rs +++ b/modules/ingestor/src/graph/cvss.rs @@ -1,5 +1,6 @@ use sea_orm::{ColumnTrait, ConnectionTrait, DbErr, EntityTrait, QueryFilter, Set}; use trustify_entity::advisory_vulnerability_score; +use trustify_entity::advisory_vulnerability_score::{ScoreType, Severity}; use uuid::Uuid; #[derive(Debug)] @@ -8,6 +9,37 @@ pub struct ScoreCreator { scores: Vec, } +/// Information required to create a new +#[derive(Clone, Debug)] +pub struct ScoreInformation { + pub vulnerability_id: String, + pub r#type: ScoreType, + pub vector: String, + pub score: f64, + pub severity: Severity, +} + +impl From for advisory_vulnerability_score::ActiveModel { + fn from(value: ScoreInformation) -> Self { + let ScoreInformation { + vulnerability_id, + r#type, + vector, + score, + severity, + } = value; + + Self { + vulnerability_id: Set(vulnerability_id), + r#type: Set(r#type), + vector: Set(vector), + score: Set(score), + severity: Set(severity), + ..Default::default() + } + } +} + impl ScoreCreator { pub fn new(advisory_id: Uuid) -> Self { Self { diff --git a/modules/ingestor/src/service/advisory/csaf/loader.rs b/modules/ingestor/src/service/advisory/csaf/loader.rs index 3b975fddb..0fd645bde 100644 --- a/modules/ingestor/src/service/advisory/csaf/loader.rs +++ b/modules/ingestor/src/service/advisory/csaf/loader.rs @@ -1,3 +1,5 @@ +use crate::graph::cvss::ScoreCreator; +use crate::service::advisory::csaf::extract_scores; use crate::{ graph::{ Graph, @@ -127,6 +129,10 @@ impl<'g> CsafLoader<'g> { .await?; } + let mut creator = ScoreCreator::new(advisory.advisory.id); + extract_scores(&csaf, &mut creator); + creator.create(&tx).await?; + tx.commit().await?; Ok(IngestResult { diff --git a/modules/ingestor/src/service/advisory/csaf/mod.rs b/modules/ingestor/src/service/advisory/csaf/mod.rs index 94f090702..7aa08172e 100644 --- a/modules/ingestor/src/service/advisory/csaf/mod.rs +++ b/modules/ingestor/src/service/advisory/csaf/mod.rs @@ -4,3 +4,36 @@ mod util; mod creator; pub use creator::*; + +use crate::graph::cvss::{ScoreCreator, ScoreInformation}; +use csaf::Csaf; +use trustify_entity::advisory_vulnerability_score::ScoreType; + +/// Extract scores from a CSAF document +pub fn extract_scores(csaf: &Csaf, creator: &mut ScoreCreator) { + for vuln in csaf.vulnerabilities.iter().flatten() { + let Some(vulnerability_id) = &vuln.cve else { + // we only process CVEs + continue; + }; + + for score in vuln.scores.iter().flatten() { + if let Some(score) = &score.cvss_v2 { + // TODO: add implementation + } + + if let Some(score) = &score.cvss_v3 { + // TODO: maybe use raw values from JSON + let vector = score.to_string(); + let score = score.score(); + creator.add(ScoreInformation { + vulnerability_id: vulnerability_id.clone(), + r#type: ScoreType::V3_0, + vector, + score: score.value(), + severity: score.severity().into(), + }) + } + } + } +} diff --git a/modules/ingestor/src/service/advisory/cve/mod.rs b/modules/ingestor/src/service/advisory/cve/mod.rs index 5ff23ffc4..9a48d35d4 100644 --- a/modules/ingestor/src/service/advisory/cve/mod.rs +++ b/modules/ingestor/src/service/advisory/cve/mod.rs @@ -4,7 +4,7 @@ use cve::Cve; pub mod divination; pub mod loader; -pub fn extract_scores(cve: &Cve, score_creator: &mut ScoreCreator) { +pub fn extract_scores(cve: &Cve, creator: &mut ScoreCreator) { let Cve::Published(cve) = cve else { return; }; From cee53717d0d7443e2620ceef27065534f348d98b Mon Sep 17 00:00:00 2001 From: Dejan Bosanac Date: Tue, 14 Oct 2025 11:18:31 +0200 Subject: [PATCH 18/47] feat: use cvss library to parse cve scores Signed-off-by: Dejan Bosanac --- Cargo.lock | 1321 +++++++++-------- Cargo.toml | 2 +- entity/Cargo.toml | 2 +- modules/ingestor/Cargo.toml | 2 +- .../ingestor/src/service/advisory/cve/mod.rs | 54 +- 5 files changed, 780 insertions(+), 601 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e995936f3..1aef2e397 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,7 +21,7 @@ dependencies = [ "actix-macros", "actix-rt", "actix_derive", - "bitflags 2.10.0", + "bitflags 2.9.4", "bytes", "crossbeam-channel", "futures-core", @@ -30,7 +30,7 @@ dependencies = [ "futures-util", "log", "once_cell", - "parking_lot 0.12.5", + "parking_lot 0.12.4", "pin-project-lite", "smallvec", "tokio", @@ -43,7 +43,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "bytes", "futures-core", "futures-sink", @@ -71,9 +71,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.11.2" +version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7926860314cbe2fb5d1f13731e387ab43bd32bca224e82e6e2db85de0a3dba49" +checksum = "44cceded2fb55f3c4b67068fa64962e2ca59614edc5b03167de9ff82ae803da0" dependencies = [ "actix-codec", "actix-rt", @@ -81,7 +81,7 @@ dependencies = [ "actix-tls", "actix-utils", "base64 0.22.1", - "bitflags 2.10.0", + "bitflags 2.9.4", "brotli", "bytes", "bytestring", @@ -116,7 +116,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -173,9 +173,9 @@ dependencies = [ [[package]] name = "actix-tls" -version = "3.5.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6176099de3f58fbddac916a7f8c6db297e021d706e7a6b99947785fee14abe9f" +checksum = "ac453898d866cdbecdbc2334fe1738c747b4eba14a677261f2b768ba05329389" dependencies = [ "actix-rt", "actix-service", @@ -271,7 +271,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -322,7 +322,16 @@ checksum = "b6ac1e58cded18cb28ddc17143c4dea5345b3ad575e14f32f66e4054a56eb271" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", +] + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", ] [[package]] @@ -355,9 +364,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -406,9 +415,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" dependencies = [ "anstyle", "anstyle-parse", @@ -421,9 +430,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" @@ -532,9 +541,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.33" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93c1f86859c1af3d514fa19e8323147ff10ea98684e6c7b307912509f50e67b2" +checksum = "9611ec0b6acea03372540509035db2f7f1e9f04da5d27728436fa994033c00a0" dependencies = [ "compression-codecs", "compression-core", @@ -562,7 +571,7 @@ dependencies = [ "futures-util", "handlebars", "http 1.3.1", - "indexmap 2.12.0", + "indexmap 2.11.4", "mime", "multer", "num-traits", @@ -610,7 +619,7 @@ dependencies = [ "proc-macro2", "quote", "strum 0.26.3", - "syn 2.0.110", + "syn 2.0.106", "thiserror 1.0.69", ] @@ -633,7 +642,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ecdaff7c9cffa3614a9f9999bf9ee4c3078fe3ce4d6a6e161736b56febf2de" dependencies = [ "bytes", - "indexmap 2.12.0", + "indexmap 2.11.4", "serde", "serde_json", ] @@ -668,7 +677,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -679,7 +688,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -705,9 +714,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.10" +version = "1.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1856b1b48b65f71a4dd940b1c0931f9a7b646d4a924b9828ffefc1454714668a" +checksum = "04b37ddf8d2e9744a0b9c19ce0b78efe4795339a90b66b7bae77987092cd2e69" dependencies = [ "aws-credential-types", "aws-runtime", @@ -735,9 +744,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.9" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86590e57ea40121d47d3f2e131bfd873dea15d78dc2f4604f4734537ad9e56c4" +checksum = "799a1290207254984cb7c05245111bc77958b92a3c9bb449598044b36341cce6" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -757,22 +766,23 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.32.3" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "107a4e9d9cab9963e04e84bb8dee0e25f2a987f9a8bad5ed054abd439caa8f8c" +checksum = "ee74396bee4da70c2e27cf94762714c911725efe69d9e2672f998512a67a4ce4" dependencies = [ "bindgen", "cc", "cmake", "dunce", "fs_extra", + "libloading", ] [[package]] name = "aws-runtime" -version = "1.5.14" +version = "1.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fe0fd441565b0b318c76e7206c8d1d0b0166b3e986cf30e890b61feb6192045" +checksum = "2e1ed337dabcf765ad5f2fb426f13af22d576328aaf09eac8f70953530798ec0" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -795,9 +805,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.112.0" +version = "1.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee73a27721035c46da0572b390a69fbdb333d0177c24f3d8f7ff952eeb96690" +checksum = "adb9118b3454ba89b30df55931a1fa7605260fc648e070b5aab402c24b375b1f" dependencies = [ "aws-credential-types", "aws-runtime", @@ -829,9 +839,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.89.0" +version = "1.85.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c1b1af02288f729e95b72bd17988c009aa72e26dcb59b3200f86d7aea726c9" +checksum = "2f2c741e2e439f07b5d1b33155e246742353d82167c785a2ff547275b7e32483" dependencies = [ "aws-credential-types", "aws-runtime", @@ -851,9 +861,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.91.0" +version = "1.87.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e8122301558dc7c6c68e878af918880b82ff41897a60c8c4e18e4dc4d93e9f1" +checksum = "6428ae5686b18c0ee99f6f3c39d94ae3f8b42894cdc35c35d8fb2470e9db2d4c" dependencies = [ "aws-credential-types", "aws-runtime", @@ -873,9 +883,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.92.0" +version = "1.87.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c7808adcff8333eaa76a849e6de926c6ac1a1268b9fd6afe32de9c29ef29d2" +checksum = "5871bec9a79a3e8d928c7788d654f135dde0e71d2dd98089388bab36b37ef607" dependencies = [ "aws-credential-types", "aws-runtime", @@ -896,9 +906,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.6" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c35452ec3f001e1f2f6db107b6373f1f48f05ec63ba2c5c9fa91f07dad32af11" +checksum = "084c34162187d39e3740cb635acd73c4e3a551a36146ad6fe8883c929c9f876c" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -924,9 +934,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.6" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "127fcfad33b7dfc531141fda7e1c402ac65f88aca5511a4d31e2e3d2cd01ce9c" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" dependencies = [ "futures-util", "pin-project-lite", @@ -935,9 +945,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.63.11" +version = "0.63.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95bd108f7b3563598e4dc7b62e1388c9982324a2abd622442167012690184591" +checksum = "56d2df0314b8e307995a3b86d44565dfe9de41f876901a7d71886c756a25979f" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -955,9 +965,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.13" +version = "0.60.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e29a304f8319781a39808847efb39561351b1bb76e933da7aa90232673638658" +checksum = "182b03393e8c677347fb5705a04a9392695d47d20ef0a2f8cfe28c8e6b9b9778" dependencies = [ "aws-smithy-types", "bytes", @@ -966,9 +976,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.5" +version = "0.62.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445d5d720c99eed0b4aa674ed00d835d9b1427dd73e04adaf2f94c6b2d6f9fca" +checksum = "7c4dacf2d38996cf729f55e7a762b30918229917eca115de45dfa8dfb97796c9" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -976,7 +986,6 @@ dependencies = [ "bytes", "bytes-utils", "futures-core", - "futures-util", "http 0.2.12", "http 1.3.1", "http-body 0.4.6", @@ -988,9 +997,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.4" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "623254723e8dfd535f566ee7b2381645f8981da086b5c4aa26c0c41582bb1d2c" +checksum = "147e8eea63a40315d704b97bf9bc9b8c1402ae94f89d5ad6f7550d963309da1b" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -1001,44 +1010,44 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.8.0", + "hyper 1.7.0", "hyper-rustls 0.24.2", "hyper-rustls 0.27.7", "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.35", - "rustls-native-certs 0.8.2", + "rustls 0.23.32", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.3", "tower", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.61.7" +version = "0.61.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2db31f727935fc63c6eeae8b37b438847639ec330a9161ece694efba257e0c54" +checksum = "eaa31b350998e703e9826b2104dd6f63be0508666e1aba88137af060e8944047" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1881b1ea6d313f9890710d65c158bdab6fb08c91ea825f74c1c8c357baf4cc" +checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.8" +version = "0.60.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d28a63441360c477465f80c7abac3b9c4d075ca638f982e605b7dc2a2c7156c9" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" dependencies = [ "aws-smithy-types", "urlencoding", @@ -1046,9 +1055,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.9.4" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bbe9d018d646b96c7be063dd07987849862b0e6d07c778aad7d93d1be6c1ef0" +checksum = "4fa63ad37685ceb7762fa4d73d06f1d5493feb88e3f27259b9ed277f4c01b185" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -1070,9 +1079,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7204f9fd94749a7c53b26da1b961b4ac36bf070ef1e0b94bb09f79d4f6c193" +checksum = "07f5e0fc8a6b3f2303f331b94504bbf754d85488f402d6f1dd7a6080f99afe56" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -1087,9 +1096,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.4" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f535879a207fce0db74b679cfc3e91a3159c8144d717d55f5832aea9eef46e" +checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" dependencies = [ "base64-simd", "bytes", @@ -1113,18 +1122,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.12" +version = "0.60.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab77cdd036b11056d2a30a7af7b775789fb024bf216acc13884c6c97752ae56" +checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.10" +version = "1.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d79fb68e3d7fe5d4833ea34dc87d2e97d26d3086cb3da660bb6b1f76d98680b6" +checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1136,15 +1145,30 @@ dependencies = [ [[package]] name = "backon" -version = "1.6.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" dependencies = [ "fastrand", "gloo-timers", "tokio", ] +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + [[package]] name = "base16ct" version = "0.1.1" @@ -1193,9 +1217,9 @@ checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "bigdecimal" -version = "0.4.9" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" +checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" dependencies = [ "autocfg", "libm", @@ -1231,7 +1255,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1242,7 +1266,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -1284,11 +1308,11 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" dependencies = [ - "serde_core", + "serde", ] [[package]] @@ -1334,9 +1358,9 @@ dependencies = [ [[package]] name = "borrow-or-share" -version = "0.2.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" +checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32" [[package]] name = "borsh" @@ -1358,7 +1382,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -1384,9 +1408,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.12.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", "serde", @@ -1461,7 +1485,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.110", + "syn 2.0.106", "zstd", ] @@ -1520,9 +1544,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c99fa31e08a43eaa5913ef68d7e01c37a2bdce6ed648168239ad33b7d30a9cd8" +checksum = "f5c434ae3cf0089ca203e9019ebe529c47ff45cefe8af7c85ecb734ef541822f" dependencies = [ "serde_core", ] @@ -1538,18 +1562,18 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.6.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" dependencies = [ "libbz2-rs-sys", ] [[package]] name = "camino" -version = "1.2.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "276a59bf2b2c967788139340c9f0c5b12d7fd6630315c15c217e559de85d2609" +checksum = "e1de8bc0aa9e9385ceb3bf0c152e3a9b9544f6c4a912c8ae504e80c1f0368603" dependencies = [ "serde_core", ] @@ -1574,7 +1598,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.16", ] [[package]] @@ -1585,9 +1609,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.45" +version = "1.2.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" +checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" dependencies = [ "find-msvc-tools", "jobserver", @@ -1606,9 +1630,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "cfg_aliases" @@ -1637,7 +1661,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -1712,9 +1736,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.51" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" +checksum = "f4512b90fa68d3a9932cea5184017c5d200f5921df706d45e853537dea51508f" dependencies = [ "clap_builder", "clap_derive", @@ -1722,9 +1746,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.51" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" +checksum = "0025e98baa12e766c67ba13ff4695a887a1eba19569aad00a472546795bd6730" dependencies = [ "anstream", "anstyle", @@ -1741,7 +1765,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -1767,9 +1791,9 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "compression-codecs" -version = "0.4.32" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "680dc087785c5230f8e8843e2e57ac7c1c90488b6a91b88caa265410568f441b" +checksum = "485abf41ac0c8047c07c87c72c8fb3eb5197f6e9d7ded615dfd1a00ae00a0f64" dependencies = [ "compression-core", "liblzma", @@ -1779,9 +1803,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.30" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9b614a5787ef0c8802a55766480563cb3a93b435898c422ed2a359cf811582" +checksum = "e47641d3deaf41fb1538ac1f54735925e275eaf3bf4d55c81b137fba797e5cbb" [[package]] name = "concurrent-queue" @@ -1802,7 +1826,7 @@ dependencies = [ "libc", "once_cell", "unicode-width", - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -1813,9 +1837,9 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const_format" -version = "0.2.35" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" dependencies = [ "const_format_proc_macros", ] @@ -1920,15 +1944,15 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc-fast" -version = "1.6.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ddc2d09feefeee8bd78101665bd8645637828fa9317f9f292496dbbd8c65ff3" +checksum = "6bf62af4cc77d8fe1c22dde4e721d87f2f54056139d8c412e1366b740305f56f" dependencies = [ "crc", "digest", + "libc", "rand 0.9.2", "regex", - "rustversion", ] [[package]] @@ -2066,11 +2090,11 @@ dependencies = [ [[package]] name = "csaf" version = "0.5.0" -source = "git+https://github.com/trustification/csaf-rs?rev=17620a225744b4a18845d4f7bf63354e01109b91#17620a225744b4a18845d4f7bf63354e01109b91" +source = "git+https://github.com/trustification/csaf-rs#17620a225744b4a18845d4f7bf63354e01109b91" dependencies = [ "chrono", "cpe", - "cvss", + "cvss 2.1.1", "packageurl", "serde", "serde_json", @@ -2099,7 +2123,7 @@ dependencies = [ "html-escape", "humantime", "log", - "parking_lot 0.12.5", + "parking_lot 0.12.4", "percent-encoding", "reqwest", "sectxtlib", @@ -2107,7 +2131,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tokio", "url", @@ -2117,21 +2141,21 @@ dependencies = [ [[package]] name = "csv" -version = "1.4.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" dependencies = [ "csv-core", "itoa", "ryu", - "serde_core", + "serde", ] [[package]] name = "csv-core" -version = "0.1.13" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" dependencies = [ "memchr", ] @@ -2160,14 +2184,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] name = "cve" -version = "0.5.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3077b0b3df7108da3ff51db5c258785866d0c2392b7dd63c554b085d3bc075cc" +checksum = "bba109f3a468c9e2cd871c259040cf11ca95caaaed10427c6c8879515ae87896" dependencies = [ "serde", "serde_json", @@ -2177,9 +2201,19 @@ dependencies = [ [[package]] name = "cvss" -version = "2.2.0" +version = "0.1.0" +source = "git+https://github.com/dejanb/cvss#aadb35f033d124da099bf74fb1383fc30a62c163" +dependencies = [ + "serde", + "serde_json", + "strum 0.26.3", +] + +[[package]] +name = "cvss" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fb220d3ce1b565af39cee5b89e47fd8dd1dab162900ee4363c8ee4169ee8a2" +checksum = "f4f643e062e9a8e26edea270945e05011c441ca6a56e9d9d4464c6b0be1352bd" dependencies = [ "serde", ] @@ -2239,7 +2273,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -2253,7 +2287,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -2275,7 +2309,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -2286,7 +2320,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -2335,9 +2369,9 @@ dependencies = [ [[package]] name = "deflate64" -version = "0.1.10" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" +checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" [[package]] name = "der" @@ -2362,9 +2396,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" dependencies = [ "powerfmt", "serde_core", @@ -2378,7 +2412,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -2424,7 +2458,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -2434,7 +2468,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core 0.20.2", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -2447,7 +2481,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -2467,7 +2501,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", "unicode-xid", ] @@ -2524,7 +2558,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -2678,7 +2712,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -2715,7 +2749,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -2823,9 +2857,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "libz-rs-sys", @@ -2970,7 +3004,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot 0.12.5", + "parking_lot 0.12.4", ] [[package]] @@ -2987,7 +3021,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -3048,7 +3082,7 @@ dependencies = [ "serde", "serde_json", "strum 0.27.2", - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "tracing", "url", @@ -3067,9 +3101,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "1.3.5" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf57c49a95fd1fe24b90b3033bee6dc7e8f1288d51494cb44e627c295e38542" +checksum = "c42bb3faf529935fbba0684910e1a71ecd271d618549d58f430b878619b7f4cf" dependencies = [ "rustversion", "typenum", @@ -3084,21 +3118,21 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.3.4" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi", - "wasip2", + "wasi 0.14.7+wasi-0.2.4", "wasm-bindgen", ] @@ -3111,16 +3145,22 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + [[package]] name = "git2" version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2deb07a133b1520dc1a5690e9bd08950108873d7ed5de38dcc74d3b5ebffa110" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "libc", "libgit2-sys", "log", @@ -3137,9 +3177,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "globset" -version = "0.4.18" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" dependencies = [ "aho-corasick", "bstr", @@ -3154,7 +3194,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "ignore", "walkdir", ] @@ -3205,7 +3245,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.12.0", + "indexmap 2.11.4", "slab", "tokio", "tokio-util", @@ -3224,7 +3264,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap 2.12.0", + "indexmap 2.11.4", "slab", "tokio", "tokio-util", @@ -3333,7 +3373,7 @@ dependencies = [ "once_cell", "rand 0.9.2", "ring", - "thiserror 2.0.17", + "thiserror 2.0.16", "tinyvec", "tokio", "tracing", @@ -3352,11 +3392,11 @@ dependencies = [ "ipconfig", "moka", "once_cell", - "parking_lot 0.12.5", + "parking_lot 0.12.4", "rand 0.9.2", "resolv-conf", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "tracing", ] @@ -3397,11 +3437,11 @@ checksum = "ad6880c8d4a9ebf39c6e8b77007ce223f646a4d21ce29d99f70cb16420545425" [[package]] name = "home" -version = "0.5.12" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3554,9 +3594,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744436df46f0bde35af3eda22aeaba453aada65d8f1c171cd8a5f59030bd69f" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" dependencies = [ "atomic-waker", "bytes", @@ -3598,15 +3638,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.3.1", - "hyper 1.8.0", + "hyper 1.7.0", "hyper-util", - "rustls 0.23.35", - "rustls-native-certs 0.8.2", + "rustls 0.23.32", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.3", "tower-service", - "webpki-roots 1.0.4", + "webpki-roots 1.0.3", ] [[package]] @@ -3615,7 +3655,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.8.0", + "hyper 1.7.0", "hyper-util", "pin-project-lite", "tokio", @@ -3630,7 +3670,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.0", + "hyper 1.7.0", "hyper-util", "native-tls", "tokio", @@ -3651,7 +3691,7 @@ dependencies = [ "futures-util", "http 1.3.1", "http-body 1.0.1", - "hyper 1.8.0", + "hyper 1.7.0", "ipnet", "libc", "percent-encoding", @@ -3676,7 +3716,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.62.0", ] [[package]] @@ -3690,9 +3730,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", "potential_utf", @@ -3703,9 +3743,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -3716,10 +3756,11 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ + "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -3730,38 +3771,42 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "2.1.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", + "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.1.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", "icu_locale_core", + "stable_deref_trait", + "tinystr", "writeable", "yoke", "zerofrom", @@ -3798,9 +3843,9 @@ dependencies = [ [[package]] name = "ignore" -version = "0.4.25" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3d782a365a015e0f5c04902246139249abf769125006fbe7649e2ee88169b4a" +checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" dependencies = [ "crossbeam-deque", "globset", @@ -3831,9 +3876,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.0" +version = "2.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", "hashbrown 0.16.0", @@ -3843,9 +3888,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.18.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9375e112e4b463ec1b1c6c011953545c65a30164fbab5b581df32b3abf0dcb88" +checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd" dependencies = [ "console", "futures-core", @@ -3874,7 +3919,7 @@ checksum = "c727f80bfa4a6c6e2508d2f05b6f4bfce242030bd88ed15ae5331c5b5d30fba7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -3898,6 +3943,17 @@ dependencies = [ "web-sys", ] +[[package]] +name = "io-uring" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +dependencies = [ + "bitflags 2.9.4", + "cfg-if", + "libc", +] + [[package]] name = "ipconfig" version = "0.3.2" @@ -3918,9 +3974,9 @@ checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "iri-string" -version = "0.7.9" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" dependencies = [ "memchr", "serde", @@ -3928,9 +3984,9 @@ dependencies = [ [[package]] name = "is_terminal_polyfill" -version = "1.70.2" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -3980,15 +4036,15 @@ version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.3.3", "libc", ] [[package]] name = "js-sys" -version = "0.3.82" +version = "0.3.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" dependencies = [ "once_cell", "wasm-bindgen", @@ -4023,7 +4079,7 @@ dependencies = [ "pest_derive", "regex", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.16", ] [[package]] @@ -4216,19 +4272,19 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.9" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-link 0.2.1", + "windows-targets 0.53.3", ] [[package]] name = "liblzma" -version = "0.4.5" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" +checksum = "10bf66f4598dc77ff96677c8e763655494f00ff9c1cf79e2eb5bb07bc31f807d" dependencies = [ "liblzma-sys", ] @@ -4256,9 +4312,9 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "libc", - "redox_syscall 0.5.18", + "redox_syscall 0.5.17", ] [[package]] @@ -4296,9 +4352,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.23" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" dependencies = [ "cc", "libc", @@ -4314,9 +4370,9 @@ checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "local-channel" @@ -4337,10 +4393,11 @@ checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" [[package]] name = "lock_api" -version = "0.4.14" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ + "autocfg", "scopeguard", ] @@ -4402,9 +4459,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memsec" @@ -4441,19 +4498,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", - "simd-adler32", ] [[package]] name = "mio" -version = "1.1.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", "log", - "wasi", - "windows-sys 0.61.2", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", ] [[package]] @@ -4466,7 +4522,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "equivalent", - "parking_lot 0.12.5", + "parking_lot 0.12.4", "portable-atomic", "rustc_version", "smallvec", @@ -4530,7 +4586,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -4545,10 +4601,11 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.6" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" dependencies = [ + "byteorder", "lazy_static", "libm", "num-integer", @@ -4644,6 +4701,15 @@ dependencies = [ "url", ] +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + [[package]] name = "oci-client" version = "0.15.0" @@ -4664,7 +4730,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "tracing", "unicase", @@ -4672,9 +4738,9 @@ dependencies = [ [[package]] name = "oci-spec" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb4684653aeaba48dea019caa17b2773e1212e281d50b6fa759f36fe032239d" +checksum = "2078e2f6be932a4de9aca90a375a45590809dfb5a08d93ab1ee217107aceeb67" dependencies = [ "const_format", "derive_builder 0.20.2", @@ -4684,7 +4750,7 @@ dependencies = [ "serde_json", "strum 0.27.2", "strum_macros 0.27.2", - "thiserror 2.0.17", + "thiserror 2.0.16", ] [[package]] @@ -4710,9 +4776,9 @@ dependencies = [ [[package]] name = "once_cell_polyfill" -version = "1.70.2" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" [[package]] name = "oorandom" @@ -4748,7 +4814,7 @@ dependencies = [ "base64 0.22.1", "biscuit", "chrono", - "getrandom 0.3.4", + "getrandom 0.3.3", "hmac-sha256", "mime", "reqwest", @@ -4792,11 +4858,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "cfg-if", "foreign-types", "libc", @@ -4813,7 +4879,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -4824,18 +4890,18 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.5.4+3.5.4" +version = "300.5.2+3.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507b3792995dae9b0df8a1c1e3771e8418b7c2d9f0baeba32e6fe8b06c7cb72" +checksum = "d270b79e2926f5150189d475bc7e9d2c69f9c4697b185fa917d5a32b792d21b4" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" dependencies = [ "cc", "libc", @@ -4854,7 +4920,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.17", + "thiserror 2.0.16", "tracing", ] @@ -4898,7 +4964,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "tonic", "tracing", @@ -4935,7 +5001,7 @@ dependencies = [ "opentelemetry", "percent-encoding", "rand 0.9.2", - "thiserror 2.0.17", + "thiserror 2.0.16", ] [[package]] @@ -4988,7 +5054,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -5105,12 +5171,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.5" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", - "parking_lot_core 0.9.12", + "parking_lot_core 0.9.11", ] [[package]] @@ -5129,15 +5195,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.12" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall 0.5.17", "smallvec", - "windows-link 0.2.1", + "windows-targets 0.52.6", ] [[package]] @@ -5230,11 +5296,12 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.3" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +checksum = "21e0a3a33733faeaf8651dfee72dd0f388f0c8e5ad496a3478fa5a922f49cfa8" dependencies = [ "memchr", + "thiserror 2.0.16", "ucd-trie", ] @@ -5262,9 +5329,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.3" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187da9a3030dbafabbbfb20cb323b976dc7b7ce91fcd84f2f74d6e31d378e2de" +checksum = "bc58706f770acb1dbd0973e6530a3cff4746fb721207feb3a8a6064cd0b6c663" dependencies = [ "pest", "pest_generator", @@ -5272,22 +5339,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.3" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b401d98f5757ebe97a26085998d6c0eecec4995cad6ab7fc30ffdf4b052843" +checksum = "6d4f36811dfe07f7b8573462465d5cb8965fffc2e71ae377a33aecf14c2c9a2f" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] name = "pest_meta" -version = "2.8.3" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f27a2cfee9f9039c4d86faa5af122a0ac3851441a34865b8a043b46be0065a" +checksum = "42919b05089acbd0a5dcd5405fb304d17d1053847b81163d09c4ad18ce8e8420" dependencies = [ "pest", "sha2", @@ -5300,18 +5367,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.12.0", + "indexmap 2.11.4", ] [[package]] name = "petgraph" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" dependencies = [ "fixedbitset 0.5.7", "hashbrown 0.15.5", - "indexmap 2.12.0", + "indexmap 2.11.4", "serde", "serde_derive", ] @@ -5380,7 +5447,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -5489,7 +5556,7 @@ dependencies = [ "tar", "target-triple", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.16", "tracing", "url", ] @@ -5500,7 +5567,7 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20698f9f0fddfa20bb0f8db60c47c7c1996b781c8e4bc2d09182d6cab66da25c" dependencies = [ - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "tracing", ] @@ -5519,7 +5586,7 @@ dependencies = [ "sqlx", "target-triple", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "tracing", "url", @@ -5527,9 +5594,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" dependencies = [ "zerovec", ] @@ -5542,9 +5609,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppmd-rust" -version = "1.3.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d558c559f0450f16f2a27a1f017ef38468c1090c9ce63c8e51366232d53717b4" +checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" [[package]] name = "ppv-lite86" @@ -5578,7 +5645,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -5618,14 +5685,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -5638,7 +5705,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", "version_check", "yansi", ] @@ -5663,7 +5730,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -5688,9 +5755,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.38.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" dependencies = [ "memchr", ] @@ -5707,9 +5774,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.35", + "rustls 0.23.32", "socket2 0.6.1", - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "tracing", "web-time", @@ -5722,15 +5789,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.3.3", "lru-slab", "rand 0.9.2", "ring", "rustc-hash", - "rustls 0.23.35", + "rustls 0.23.32", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.16", "tinyvec", "tracing", "web-time", @@ -5752,9 +5819,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.42" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -5827,7 +5894,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.3.3", ] [[package]] @@ -5861,11 +5928,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.18" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", ] [[package]] @@ -5881,22 +5948,22 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.25" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.25" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -5965,7 +6032,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.7.0", "hyper-rustls 0.27.7", "hyper-tls", "hyper-util", @@ -5976,8 +6043,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", - "rustls-native-certs 0.8.2", + "rustls 0.23.32", + "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", "serde_json", @@ -5985,7 +6052,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.3", "tokio-util", "tower", "tower-http", @@ -5995,7 +6062,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.4", + "webpki-roots 1.0.3", ] [[package]] @@ -6024,7 +6091,7 @@ dependencies = [ "futures", "getrandom 0.2.16", "http 1.3.1", - "hyper 1.8.0", + "hyper 1.7.0", "parking_lot 0.11.2", "reqwest", "reqwest-middleware", @@ -6203,7 +6270,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.110", + "syn 2.0.106", "unicode-ident", ] @@ -6220,9 +6287,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "8.9.0" +version = "8.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947d7f3fad52b283d261c4c99a084937e2fe492248cb9a68a8435a861b8798ca" +checksum = "025908b8682a26ba8d12f6f2d66b987584a4a87bc024abc5bbc12553a8cd178a" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -6231,22 +6298,22 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "8.9.0" +version = "8.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa2c8c9e8711e10f9c4fd2d64317ef13feaab820a4c51541f1a8c8e2e851ab2" +checksum = "6065f1a4392b71819ec1ea1df1120673418bf386f50de1d6f54204d836d4349c" dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.110", + "syn 2.0.106", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "8.9.0" +version = "8.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b161f275cb337fe0a44d924a5f4df0ed69c2c39519858f931ce61c779d3475" +checksum = "f6cc0c81648b20b70c491ff8cce00c1c3b223bb8ed2b5d41f0e54c6c4c0a3594" dependencies = [ "sha2", "walkdir", @@ -6268,6 +6335,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -6289,11 +6362,11 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -6310,15 +6383,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40" dependencies = [ "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.8", + "rustls-webpki 0.103.6", "subtle", "zeroize", ] @@ -6337,14 +6410,14 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.2" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework 3.5.0", ] [[package]] @@ -6358,9 +6431,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ "web-time", "zeroize", @@ -6378,9 +6451,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.8" +version = "0.103.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +checksum = "8572f3c2cb9934231157b45499fc41e1f58c589fdfb81a844ba873265e80f8eb" dependencies = [ "aws-lc-rs", "ring", @@ -6432,7 +6505,7 @@ dependencies = [ "futures", "humantime", "log", - "parking_lot 0.12.5", + "parking_lot 0.12.4", "reqwest", "sequoia-openpgp", "serde", @@ -6440,7 +6513,7 @@ dependencies = [ "serde_json", "sha2", "spdx-rs", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tokio", "url", @@ -6453,7 +6526,7 @@ version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -6510,13 +6583,13 @@ dependencies = [ [[package]] name = "schemars" -version = "1.1.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" +checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" dependencies = [ "dyn-clone", "ref-cast", - "schemars_derive 1.1.0", + "schemars_derive 1.0.4", "serde", "serde_json", "url", @@ -6531,19 +6604,19 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] name = "schemars_derive" -version = "1.1.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" +checksum = "33d020396d1d138dc19f1165df7545479dcd58d93810dc5d646a16e55abefa80" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -6572,14 +6645,14 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] name = "sea-orm" -version = "1.1.19" +version = "1.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d945f62558fac19e5988680d2fdf747b734c2dbc6ce2cb81ba33ed8dde5b103" +checksum = "699b1ec145a6530c8f862eed7529d8a6068392e628d81cc70182934001e9c2a3" dependencies = [ "async-stream", "async-trait", @@ -6598,7 +6671,7 @@ dependencies = [ "serde_json", "sqlx", "strum 0.26.3", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tracing", "url", @@ -6607,9 +6680,9 @@ dependencies = [ [[package]] name = "sea-orm-cli" -version = "1.1.19" +version = "1.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c94492e2ab6c045b4cc38013809ce255d14c3d352c9f0d11e6b920e2adc948ad" +checksum = "500cd31ebb07814d4c7b73796708bfab6c13d22f8db072cdb5115f967f4d5d2c" dependencies = [ "chrono", "clap", @@ -6626,23 +6699,23 @@ dependencies = [ [[package]] name = "sea-orm-macros" -version = "1.1.19" +version = "1.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c2e64a50a9cc8339f10a27577e10062c7f995488e469f2c95762c5ee847832" +checksum = "b0c964f4b7f34f53decf381bc88f03187b9355e07f356ce65544626e781a9585" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", "sea-bae", - "syn 2.0.110", + "syn 2.0.106", "unicode-ident", ] [[package]] name = "sea-orm-migration" -version = "1.1.19" +version = "1.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7315c0cadb7e60fb17ee2bb282aa27d01911fc2a7e5836ec1d4ac37d19250bb4" +checksum = "977e3f71486b04371026d1ecd899f49cf437f832cd11d463f8948ee02e47ed9e" dependencies = [ "async-trait", "clap", @@ -6697,8 +6770,8 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.110", - "thiserror 2.0.17", + "syn 2.0.106", + "thiserror 2.0.16", ] [[package]] @@ -6723,7 +6796,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -6770,7 +6843,7 @@ dependencies = [ "iri-string", "nom", "oxilangtag", - "thiserror 2.0.17", + "thiserror 2.0.16", "valuable", ] @@ -6780,7 +6853,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -6789,11 +6862,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.5.1" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "cc198e42d9b7510827939c9a15f5062a0c913f3371d765977e586d2fe6c16f4a" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -6822,9 +6895,9 @@ dependencies = [ [[package]] name = "sequoia-openpgp" -version = "2.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0e334ce3ec5b9b47d86a80563b3ecec435f59acf37e86058b3b686a42c5a2ba" +checksum = "015e5fc3d023418b9db98ca9a7f3e90b305872eeafe5ca45c5c32b5eb335c1e8" dependencies = [ "anyhow", "argon2", @@ -6843,15 +6916,15 @@ dependencies = [ "regex", "regex-syntax", "sha1collisiondetection", - "thiserror 2.0.17", + "thiserror 2.0.16", "xxhash-rust", ] [[package]] name = "serde" -version = "1.0.228" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" dependencies = [ "serde_core", "serde_derive", @@ -6871,7 +6944,7 @@ dependencies = [ "schemafy_lib", "serde", "serde_json", - "syn 2.0.110", + "syn 2.0.106", "thiserror 1.0.69", ] @@ -6887,22 +6960,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -6913,7 +6986,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -6922,7 +6995,7 @@ version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.11.4", "itoa", "memchr", "ryu", @@ -6936,7 +7009,7 @@ version = "0.9.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e408f29489b5fd500fab51ff1484fc859bb655f32c671f307dcd733b72e8168c" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.11.4", "itoa", "ryu", "serde", @@ -6977,18 +7050,19 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.15.1" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04" +checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.0", + "indexmap 2.11.4", "schemars 0.9.0", - "schemars 1.1.0", - "serde_core", + "schemars 1.0.4", + "serde", + "serde_derive", "serde_json", "serde_with_macros", "time", @@ -6996,14 +7070,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.15.1" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955" +checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -7012,7 +7086,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.11.4", "itoa", "ryu", "serde", @@ -7043,7 +7117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f606421e4a6012877e893c399822a4ed4b089164c5969424e1b9d1e66e6964b" dependencies = [ "digest", - "generic-array 1.3.5", + "generic-array 1.3.3", ] [[package]] @@ -7262,19 +7336,19 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap 2.12.0", + "indexmap 2.11.4", "log", "memchr", "native-tls", "once_cell", "percent-encoding", "rust_decimal", - "rustls 0.23.35", + "rustls 0.23.32", "serde", "serde_json", "sha2", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tokio", "tokio-stream", @@ -7294,7 +7368,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -7317,7 +7391,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.110", + "syn 2.0.106", "tokio", "url", ] @@ -7331,7 +7405,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.10.0", + "bitflags 2.9.4", "byteorder", "bytes", "chrono", @@ -7362,7 +7436,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tracing", "uuid", @@ -7378,7 +7452,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.10.0", + "bitflags 2.9.4", "byteorder", "chrono", "crc", @@ -7405,7 +7479,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tracing", "uuid", @@ -7432,7 +7506,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tracing", "url", @@ -7486,7 +7560,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" dependencies = [ "new_debug_unreachable", - "parking_lot 0.12.5", + "parking_lot 0.12.4", "phf_shared", "precomputed-hash", ] @@ -7542,7 +7616,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -7555,7 +7629,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -7567,7 +7641,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -7589,9 +7663,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.110" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -7615,7 +7689,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -7624,7 +7698,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -7674,7 +7748,7 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050" dependencies = [ - "parking_lot 0.12.5", + "parking_lot 0.12.4", ] [[package]] @@ -7684,17 +7758,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] name = "tera" -version = "1.20.1" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8004bca281f2d32df3bacd59bc67b312cb4c70cea46cbd79dbe8ac5ed206722" +checksum = "ab9d851b45e865f178319da0abdbfe6acbc4328759ff18dafc3a41c16b4cd2ee" dependencies = [ "chrono", "chrono-tz", @@ -7709,7 +7783,7 @@ dependencies = [ "serde", "serde_json", "slug", - "unicode-segmentation", + "unic-segment", ] [[package]] @@ -7725,9 +7799,9 @@ dependencies = [ [[package]] name = "test-context" -version = "0.5.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d94db16dc1c321805ce55f286c4023fa58a2c9c742568f95c5cfe2e95d250d7" +checksum = "cb69cce03e432993e2dc1f93f7899b952300fcb6dc44191a1b830b60b8c3c8aa" dependencies = [ "futures", "test-context-macros", @@ -7735,13 +7809,13 @@ dependencies = [ [[package]] name = "test-context-macros" -version = "0.5.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aabcca9d2cad192cfe258cd3562b7584516191a5c9b6a0002a6bb8b75ee7d21d" +checksum = "97e0639209021e54dbe19cafabfc0b5574b078c37358945e6d473eabe39bb974" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -7763,7 +7837,7 @@ checksum = "451b374529930d7601b1eef8d32bc79ae870b6079b069401709c2a8bf9e75f36" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -7777,11 +7851,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.16", ] [[package]] @@ -7792,18 +7866,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -7865,9 +7939,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -7900,30 +7974,33 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ + "backtrace", "bytes", + "io-uring", "libc", "mio", - "parking_lot 0.12.5", + "parking_lot 0.12.4", "pin-project-lite", "signal-hook-registry", + "slab", "socket2 0.6.1", "tokio-macros", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -7959,11 +8036,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "05f63835928ca123f1bef57abbcd23bb2ba0ac9ae1235f1e65bda0d06e7786bd" dependencies = [ - "rustls 0.23.35", + "rustls 0.23.32", "tokio", ] @@ -7980,9 +8057,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" dependencies = [ "bytes", "futures-core", @@ -8020,7 +8097,7 @@ version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.11.4", "toml_datetime", "toml_parser", "winnow", @@ -8047,7 +8124,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.7.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -8080,7 +8157,7 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", - "indexmap 2.12.0", + "indexmap 2.11.4", "pin-project-lite", "slab", "sync_wrapper", @@ -8097,7 +8174,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.9.4", "bytes", "futures-util", "http 1.3.1", @@ -8141,7 +8218,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -8199,7 +8276,7 @@ dependencies = [ "opentelemetry_sdk", "rustversion", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.16", "tracing", "tracing-core", "tracing-log", @@ -8245,12 +8322,12 @@ dependencies = [ "log", "openid 0.18.3", "reqwest", - "schemars 1.1.0", + "schemars 1.0.4", "serde", "serde_json", "serde_yaml_ng", "strum 0.27.2", - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "tracing", "trustify-common", @@ -8287,7 +8364,7 @@ dependencies = [ "ring", "rstest", "sbom-walker", - "schemars 1.1.0", + "schemars 1.0.4", "sea-orm", "sea-orm-migration", "sea-query", @@ -8299,7 +8376,7 @@ dependencies = [ "sqlx", "strum 0.27.2", "test-log", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tokio", "tracing", @@ -8344,18 +8421,18 @@ dependencies = [ "anyhow", "async-graphql", "cpe", - "cvss", + "cvss 0.1.0", "deepsize", "log", "rstest", - "schemars 1.1.0", + "schemars 1.0.4", "sea-orm", "serde", "serde_json", "strum 0.27.2", "test-context", "test-log", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tokio", "trustify-common", @@ -8385,7 +8462,7 @@ dependencies = [ "opentelemetry-instrumentation-actix-web", "opentelemetry-otlp", "opentelemetry_sdk", - "parking_lot 0.12.5", + "parking_lot 0.12.4", "reqwest", "serde", "serde_json", @@ -8460,9 +8537,8 @@ dependencies = [ "moka", "opentelemetry", "packageurl", - "parking_lot 0.12.5", - "petgraph 0.8.3", - "rstest", + "parking_lot 0.12.4", + "petgraph 0.8.2", "sea-orm", "sea-query", "serde", @@ -8470,7 +8546,7 @@ dependencies = [ "sha2", "test-context", "test-log", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tokio", "tokio-util", @@ -8536,7 +8612,7 @@ dependencies = [ "tar", "test-context", "test-log", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tokio", "tokio-util", @@ -8608,11 +8684,11 @@ dependencies = [ "oci-client", "opentelemetry", "osv", - "parking_lot 0.12.5", + "parking_lot 0.12.4", "regex", "reqwest", "sbom-walker", - "schemars 1.1.0", + "schemars 1.0.4", "sea-orm", "sea-query", "serde", @@ -8621,7 +8697,7 @@ dependencies = [ "tempfile", "test-context", "test-log", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tokio", "tokio-util", @@ -8655,7 +8731,7 @@ dependencies = [ "cpe", "csaf", "cve", - "cvss", + "cvss 0.1.0", "hex", "humantime", "jsn", @@ -8664,7 +8740,7 @@ dependencies = [ "log", "osv", "packageurl", - "parking_lot 0.12.5", + "parking_lot 0.12.4", "quick-xml", "rand 0.9.2", "roxmltree", @@ -8682,7 +8758,7 @@ dependencies = [ "strum 0.27.2", "test-context", "test-log", - "thiserror 2.0.17", + "thiserror 2.0.16", "time", "tokio", "tracing", @@ -8721,7 +8797,7 @@ dependencies = [ "tempfile", "test-context", "test-log", - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "tokio-util", "tracing", @@ -8742,7 +8818,7 @@ dependencies = [ "serde_json", "spdx-rs", "test-log", - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "trustify-common", "trustify-module-ingestor", @@ -8764,7 +8840,7 @@ dependencies = [ "serde_json", "test-context", "test-log", - "thiserror 2.0.17", + "thiserror 2.0.16", "tokio", "trustify-auth", "trustify-common", @@ -8786,7 +8862,7 @@ name = "trustify-query-derive" version = "0.4.0-beta.1" dependencies = [ "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -8897,7 +8973,7 @@ dependencies = [ [[package]] name = "trustify-ui" version = "0.1.0" -source = "git+https://github.com/guacsec/trustify-ui.git?branch=publish%2Fmain#4e1c3cbef45f7b891b3777256b75f1c2f3cff69b" +source = "git+https://github.com/guacsec/trustify-ui.git?branch=publish%2Fmain#5e64b9adcc138efb1b4cc21cfc60d19a46e8e174" dependencies = [ "anyhow", "base64 0.22.1", @@ -8916,9 +8992,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.19.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "ucd-trie" @@ -8926,6 +9002,56 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +dependencies = [ + "unic-char-range", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" + +[[package]] +name = "unic-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" +dependencies = [ + "unic-ucd-segment", +] + +[[package]] +name = "unic-ucd-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +dependencies = [ + "unic-common", +] + [[package]] name = "unicase" version = "2.8.1" @@ -8940,36 +9066,30 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" [[package]] name = "unicode-normalization" -version = "0.1.25" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-properties" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" [[package]] name = "unicode-width" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "unicode-xid" @@ -9059,7 +9179,7 @@ version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fcc29c80c21c31608227e0912b2d7fddba57ad76b606890627ba8ee7964e993" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.11.4", "serde", "serde_json", "serde_norway", @@ -9086,7 +9206,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.110", + "syn 2.0.106", "url", "uuid", ] @@ -9139,7 +9259,7 @@ version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.3.3", "js-sys", "serde", "sha1_smol", @@ -9173,7 +9293,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -9193,7 +9313,7 @@ checksum = "4e3a32a9bcc0f6c6ccfd5b27bcf298c58e753bcc9eeff268157a303393183a6d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -9266,7 +9386,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "thiserror 2.0.17", + "thiserror 2.0.16", "thousands", "time", "tokio", @@ -9290,6 +9410,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" @@ -9307,9 +9436,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.105" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" dependencies = [ "cfg-if", "once_cell", @@ -9318,11 +9447,25 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn 2.0.106", + "wasm-bindgen-shared", +] + [[package]] name = "wasm-bindgen-futures" -version = "0.4.55" +version = "0.4.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" +checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c" dependencies = [ "cfg-if", "js-sys", @@ -9333,9 +9476,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.105" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9343,22 +9486,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.105" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" dependencies = [ - "bumpalo", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", + "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.105" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" dependencies = [ "unicode-ident", ] @@ -9393,9 +9536,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.82" +version = "0.3.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" +checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" dependencies = [ "js-sys", "wasm-bindgen", @@ -9417,14 +9560,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.4", + "webpki-roots 1.0.3", ] [[package]] name = "webpki-roots" -version = "1.0.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8" dependencies = [ "rustls-pki-types", ] @@ -9467,7 +9610,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -9500,15 +9643,15 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.62.2" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c" dependencies = [ - "windows-implement 0.60.2", - "windows-interface 0.59.3", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", + "windows-implement 0.60.0", + "windows-interface 0.59.1", + "windows-link 0.2.0", + "windows-result 0.4.0", + "windows-strings 0.5.0", ] [[package]] @@ -9519,18 +9662,18 @@ checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] name = "windows-implement" -version = "0.60.2" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -9541,18 +9684,18 @@ checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] name = "windows-interface" -version = "0.59.3" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -9563,9 +9706,9 @@ checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-link" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" [[package]] name = "windows-registry" @@ -9598,11 +9741,11 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f" dependencies = [ - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -9616,11 +9759,11 @@ dependencies = [ [[package]] name = "windows-strings" -version = "0.5.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda" dependencies = [ - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -9641,22 +9784,31 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.5", + "windows-targets 0.53.3", ] [[package]] name = "windows-sys" -version = "0.61.2" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" dependencies = [ - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -9692,19 +9844,19 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.5" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ - "windows-link 0.2.1", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows-link 0.1.3", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -9721,9 +9873,9 @@ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_gnullvm" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" [[package]] name = "windows_aarch64_msvc" @@ -9739,9 +9891,9 @@ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_aarch64_msvc" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" [[package]] name = "windows_i686_gnu" @@ -9757,9 +9909,9 @@ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnu" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" [[package]] name = "windows_i686_gnullvm" @@ -9769,9 +9921,9 @@ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_gnullvm" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" [[package]] name = "windows_i686_msvc" @@ -9787,9 +9939,9 @@ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_i686_msvc" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" [[package]] name = "windows_x86_64_gnu" @@ -9805,9 +9957,9 @@ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnu" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" [[package]] name = "windows_x86_64_gnullvm" @@ -9823,9 +9975,9 @@ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_gnullvm" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" [[package]] name = "windows_x86_64_msvc" @@ -9841,9 +9993,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "windows_x86_64_msvc" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" @@ -9876,7 +10028,7 @@ dependencies = [ "futures", "http 1.3.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.7.0", "hyper-util", "log", "once_cell", @@ -9895,9 +10047,9 @@ checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "wyz" @@ -9933,7 +10085,7 @@ dependencies = [ "clap", "log", "postgresql_commands", - "schemars 1.1.0", + "schemars 1.0.4", "serde", "serde_json", "serde_yaml_ng", @@ -9965,10 +10117,11 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ + "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -9976,13 +10129,13 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", "synstructure", ] @@ -10003,7 +10156,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -10023,15 +10176,15 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] @@ -10044,14 +10197,14 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" dependencies = [ "displaydoc", "yoke", @@ -10060,9 +10213,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" dependencies = [ "yoke", "zerofrom", @@ -10071,13 +10224,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] @@ -10089,7 +10242,7 @@ dependencies = [ "arbitrary", "crc32fast", "flate2", - "indexmap 2.12.0", + "indexmap 2.11.4", "memchr", "zopfli", ] @@ -10107,9 +10260,9 @@ dependencies = [ "crc32fast", "deflate64", "flate2", - "getrandom 0.3.4", + "getrandom 0.3.3", "hmac", - "indexmap 2.12.0", + "indexmap 2.11.4", "lzma-rust2", "memchr", "pbkdf2", @@ -10129,9 +10282,9 @@ checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" [[package]] name = "zopfli" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" dependencies = [ "bumpalo", "crc32fast", diff --git a/Cargo.toml b/Cargo.toml index e1708d09d..9d65ba3ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ csaf = { version = "0.5.0", default-features = false } csaf-walker = { version = "0.14.1", default-features = false } csv = "1.3.0" cve = "0.5.0" -cvss = "2" +cvss-rs = "0.2" deepsize = "0.2.0" fixedbitset = "0.5.7" flate2 = "1.0.35" diff --git a/entity/Cargo.toml b/entity/Cargo.toml index f0e217657..574819b75 100644 --- a/entity/Cargo.toml +++ b/entity/Cargo.toml @@ -13,7 +13,7 @@ trustify-common = { workspace = true } trustify-cvss = { workspace = true } cpe = { workspace = true } -cvss = { workspace = true } +cvss-rs = { workspace = true } deepsize = { workspace = true } schemars = { workspace = true } sea-orm = { workspace = true, features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json", "postgres-array"] } diff --git a/modules/ingestor/Cargo.toml b/modules/ingestor/Cargo.toml index 4fe24e7f3..1485c0e11 100644 --- a/modules/ingestor/Cargo.toml +++ b/modules/ingestor/Cargo.toml @@ -19,7 +19,7 @@ bytes = { workspace = true } cpe = { workspace = true } csaf = { workspace = true } cve = { workspace = true } -cvss = { workspace = true } +cvss-rs = { workspace = true } hex = { workspace = true } humantime = { workspace = true } jsn = { workspace = true } diff --git a/modules/ingestor/src/service/advisory/cve/mod.rs b/modules/ingestor/src/service/advisory/cve/mod.rs index 9a48d35d4..f0b6ad201 100644 --- a/modules/ingestor/src/service/advisory/cve/mod.rs +++ b/modules/ingestor/src/service/advisory/cve/mod.rs @@ -1,26 +1,52 @@ use crate::graph::cvss::ScoreCreator; use cve::Cve; +use cvss::{Cvss, v2_0::CvssV2, v3::CvssV3, v4_0::CvssV4}; pub mod divination; pub mod loader; -pub fn extract_scores(cve: &Cve, creator: &mut ScoreCreator) { - let Cve::Published(cve) = cve else { +pub fn extract_scores(cve: &Cve, _score_creator: &mut ScoreCreator) { + let Cve::Published(published) = cve else { return; }; - for metrics in &cve.containers.cna.metrics { - if let Some(value) = &metrics.cvss_v2_0 { - // TODO: add score to creator - } - if let Some(value) = &metrics.cvss_v3_0 { - // TODO: add score to creator - } - if let Some(value) = &metrics.cvss_v3_1 { - // TODO: add score to creator - } - if let Some(value) = &metrics.cvss_v4_0 { - // TODO: add score to creator + let all_metrics = published.containers.cna.metrics.iter().chain( + published + .containers + .adp + .iter() + .flat_map(|adp| adp.metrics.iter()), + ); + + for metric in all_metrics { + let cvss_objects: Vec = [ + metric.cvss_v2_0.as_ref().and_then(|v| { + serde_json::from_value::(v.clone()) + .ok() + .map(Cvss::V2) + }), + metric.cvss_v3_0.as_ref().and_then(|v| { + serde_json::from_value::(v.clone()) + .ok() + .map(Cvss::V3_0) + }), + metric.cvss_v3_1.as_ref().and_then(|v| { + serde_json::from_value::(v.clone()) + .ok() + .map(Cvss::V3_1) + }), + metric.cvss_v4_0.as_ref().and_then(|v| { + serde_json::from_value::(v.clone()) + .ok() + .map(Cvss::V4) + }), + ] + .into_iter() + .flatten() + .collect(); + + for cvss in cvss_objects { + println!("Parsed CVSS: {:?}", cvss); } } } From b29d94c9e89420f68978d9e02b59ce00872b6674 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 15 Oct 2025 12:17:49 +0200 Subject: [PATCH 19/47] chore: mop up some things --- Cargo.lock | 1 + Cargo.toml | 1 + entity/Cargo.toml | 1 + entity/src/advisory_vulnerability_score.rs | 12 ++++++++++++ 4 files changed, 15 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 1aef2e397..297daa5ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8422,6 +8422,7 @@ dependencies = [ "async-graphql", "cpe", "cvss 0.1.0", + "cvss 2.1.1", "deepsize", "log", "rstest", diff --git a/Cargo.toml b/Cargo.toml index 9d65ba3ab..dbd98d13a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ csaf-walker = { version = "0.14.1", default-features = false } csv = "1.3.0" cve = "0.5.0" cvss-rs = "0.2" +cvss-old = { package = "cvss", version = "2" } deepsize = "0.2.0" fixedbitset = "0.5.7" flate2 = "1.0.35" diff --git a/entity/Cargo.toml b/entity/Cargo.toml index 574819b75..c409e49f1 100644 --- a/entity/Cargo.toml +++ b/entity/Cargo.toml @@ -14,6 +14,7 @@ trustify-cvss = { workspace = true } cpe = { workspace = true } cvss-rs = { workspace = true } +cvss-old = { workspace = true } deepsize = { workspace = true } schemars = { workspace = true } sea-orm = { workspace = true, features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json", "postgres-array"] } diff --git a/entity/src/advisory_vulnerability_score.rs b/entity/src/advisory_vulnerability_score.rs index 7bb4e71c5..b3332323c 100644 --- a/entity/src/advisory_vulnerability_score.rs +++ b/entity/src/advisory_vulnerability_score.rs @@ -137,3 +137,15 @@ impl From for Severity { } } } + +impl From for Severity { + fn from(value: cvss_old::Severity) -> Self { + match value { + cvss_old::Severity::None => Self::None, + cvss_old::Severity::Low => Self::Low, + cvss_old::Severity::Medium => Self::Medium, + cvss_old::Severity::High => Self::High, + cvss_old::Severity::Critical => Self::Critical, + } + } +} From d6f22921dc6a6886765a57754ee58617d3b405d3 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 15 Oct 2025 15:06:42 +0200 Subject: [PATCH 20/47] chore: refactor code about ingesting new scores --- entity/src/advisory_vulnerability_score.rs | 25 ++++ modules/ingestor/src/graph/cvss.rs | 127 +++++++++++++++--- .../ingestor/src/service/advisory/cve/mod.rs | 56 ++++---- .../ingestor/src/service/advisory/osv/mod.rs | 100 ++++++++------ 4 files changed, 222 insertions(+), 86 deletions(-) diff --git a/entity/src/advisory_vulnerability_score.rs b/entity/src/advisory_vulnerability_score.rs index b3332323c..1cb1b53df 100644 --- a/entity/src/advisory_vulnerability_score.rs +++ b/entity/src/advisory_vulnerability_score.rs @@ -1,4 +1,5 @@ use crate::{advisory, advisory_vulnerability, cvss3, vulnerability}; +use cvss::{v3, v4_0}; use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel)] @@ -149,3 +150,27 @@ impl From for Severity { } } } + +impl From for Severity { + fn from(value: v3::Severity) -> Self { + match value { + v3::Severity::None => Self::None, + v3::Severity::Low => Self::Low, + v3::Severity::Medium => Self::Medium, + v3::Severity::High => Self::High, + v3::Severity::Critical => Self::Critical, + } + } +} + +impl From for Severity { + fn from(value: v4_0::Severity) -> Self { + match value { + v4_0::Severity::None => Self::None, + v4_0::Severity::Low => Self::Low, + v4_0::Severity::Medium => Self::Medium, + v4_0::Severity::High => Self::High, + v4_0::Severity::Critical => Self::Critical, + } + } +} diff --git a/modules/ingestor/src/graph/cvss.rs b/modules/ingestor/src/graph/cvss.rs index ef28d54ec..ac782465c 100644 --- a/modules/ingestor/src/graph/cvss.rs +++ b/modules/ingestor/src/graph/cvss.rs @@ -1,12 +1,13 @@ +use cvss::version::{VersionV2, VersionV3, VersionV4}; +use cvss::{Cvss, v2_0, v3, v4_0}; use sea_orm::{ColumnTrait, ConnectionTrait, DbErr, EntityTrait, QueryFilter, Set}; -use trustify_entity::advisory_vulnerability_score; -use trustify_entity::advisory_vulnerability_score::{ScoreType, Severity}; +use trustify_entity::advisory_vulnerability_score::{self, ScoreType, Severity}; use uuid::Uuid; #[derive(Debug)] pub struct ScoreCreator { advisory_id: Uuid, - scores: Vec, + scores: Vec, } /// Information required to create a new @@ -40,6 +41,89 @@ impl From for advisory_vulnerability_score::ActiveModel { } } +impl From<(String, v2_0::CvssV2)> for ScoreInformation { + fn from((vulnerability_id, score): (String, v2_0::CvssV2)) -> Self { + let v2_0::CvssV2 { + version, + vector_string, + severity, + base_score, + .. + } = score; + + Self { + vulnerability_id, + r#type: match version { + VersionV2::V2_0 => ScoreType::V2_0, + }, + vector: vector_string, + score: base_score, + severity: match severity { + None => Severity::None, + Some(v2_0::Severity::Low) => Severity::Low, + Some(v2_0::Severity::Medium) => Severity::Medium, + Some(v2_0::Severity::High) => Severity::High, + }, + } + } +} + +impl From<(String, v3::CvssV3)> for ScoreInformation { + fn from((vulnerability_id, score): (String, v3::CvssV3)) -> Self { + let v3::CvssV3 { + version, + vector_string, + base_severity, + base_score, + .. + } = score; + + Self { + vulnerability_id, + r#type: match version { + VersionV3::V3_0 => ScoreType::V3_0, + VersionV3::V3_1 => ScoreType::V3_1, + }, + vector: vector_string, + score: base_score, + severity: base_severity.into(), + } + } +} + +impl From<(String, v4_0::CvssV4)> for ScoreInformation { + fn from((vulnerability_id, score): (String, v4_0::CvssV4)) -> Self { + let v4_0::CvssV4 { + version, + vector_string, + base_severity, + base_score, + .. + } = score; + + Self { + vulnerability_id, + r#type: match version { + VersionV4::V4_0 => ScoreType::V4_0, + }, + vector: vector_string, + score: base_score, + severity: base_severity.into(), + } + } +} + +impl From<(String, Cvss)> for ScoreInformation { + fn from((vulnerability_id, score): (String, Cvss)) -> Self { + match score { + Cvss::V2(score) => (vulnerability_id, score).into(), + Cvss::V3_0(score) => (vulnerability_id, score).into(), + Cvss::V3_1(score) => (vulnerability_id, score).into(), + Cvss::V4(score) => (vulnerability_id, score).into(), + } + } +} + impl ScoreCreator { pub fn new(advisory_id: Uuid) -> Self { Self { @@ -48,15 +132,12 @@ impl ScoreCreator { } } - pub fn add(&mut self, model: impl Into) { + pub fn add(&mut self, model: impl Into) { self.scores.push(model.into()); } - pub fn extend( - &mut self, - items: impl IntoIterator, - ) { - self.scores.extend(items) + pub fn extend(&mut self, items: impl IntoIterator>) { + self.scores.extend(items.into_iter().map(Into::into)); } pub async fn create(self, db: &C) -> Result<(), DbErr> @@ -65,7 +146,7 @@ impl ScoreCreator { { let Self { advisory_id, - mut scores, + scores, } = self; // delete existing entries @@ -75,11 +156,27 @@ impl ScoreCreator { .exec(db) .await?; - // set advisory - - for score in &mut scores { - score.advisory_id = Set(self.advisory_id); - } + // transform and set advisory + + let scores = scores.into_iter().map(|score| { + let ScoreInformation { + vulnerability_id, + r#type, + vector, + score, + severity, + } = score; + + advisory_vulnerability_score::ActiveModel { + id: Set(Uuid::now_v7()), + advisory_id: Set(advisory_id), + vulnerability_id: Set(vulnerability_id), + r#type: Set(r#type), + vector: Set(vector), + score: Set(score), + severity: Set(severity), + } + }); // insert chunked diff --git a/modules/ingestor/src/service/advisory/cve/mod.rs b/modules/ingestor/src/service/advisory/cve/mod.rs index f0b6ad201..f49b7f074 100644 --- a/modules/ingestor/src/service/advisory/cve/mod.rs +++ b/modules/ingestor/src/service/advisory/cve/mod.rs @@ -1,11 +1,11 @@ use crate::graph::cvss::ScoreCreator; use cve::Cve; -use cvss::{Cvss, v2_0::CvssV2, v3::CvssV3, v4_0::CvssV4}; +use cvss::{v2_0::CvssV2, v3::CvssV3, v4_0::CvssV4}; pub mod divination; pub mod loader; -pub fn extract_scores(cve: &Cve, _score_creator: &mut ScoreCreator) { +pub fn extract_scores(cve: &Cve, creator: &mut ScoreCreator) { let Cve::Published(published) = cve else { return; }; @@ -18,35 +18,31 @@ pub fn extract_scores(cve: &Cve, _score_creator: &mut ScoreCreator) { .flat_map(|adp| adp.metrics.iter()), ); + let vulnerability_id = &published.metadata.id; + for metric in all_metrics { - let cvss_objects: Vec = [ - metric.cvss_v2_0.as_ref().and_then(|v| { - serde_json::from_value::(v.clone()) - .ok() - .map(Cvss::V2) - }), - metric.cvss_v3_0.as_ref().and_then(|v| { - serde_json::from_value::(v.clone()) - .ok() - .map(Cvss::V3_0) - }), - metric.cvss_v3_1.as_ref().and_then(|v| { - serde_json::from_value::(v.clone()) - .ok() - .map(Cvss::V3_1) - }), - metric.cvss_v4_0.as_ref().and_then(|v| { - serde_json::from_value::(v.clone()) - .ok() - .map(Cvss::V4) - }), - ] - .into_iter() - .flatten() - .collect(); - - for cvss in cvss_objects { - println!("Parsed CVSS: {:?}", cvss); + if let Some(v) = &metric.cvss_v2_0 + && let Ok(score) = serde_json::from_value::(v.clone()) + { + creator.add((vulnerability_id.clone(), score)); + } + + if let Some(v) = &metric.cvss_v3_0 + && let Ok(score) = serde_json::from_value::(v.clone()) + { + creator.add((vulnerability_id.clone(), score)); + } + + if let Some(v) = &metric.cvss_v3_1 + && let Ok(score) = serde_json::from_value::(v.clone()) + { + creator.add((vulnerability_id.clone(), score)); + } + + if let Some(v) = &metric.cvss_v4_0 + && let Ok(score) = serde_json::from_value::(v.clone()) + { + creator.add((vulnerability_id.clone(), score)); } } } diff --git a/modules/ingestor/src/service/advisory/osv/mod.rs b/modules/ingestor/src/service/advisory/osv/mod.rs index d776c802f..51e9a602f 100644 --- a/modules/ingestor/src/service/advisory/osv/mod.rs +++ b/modules/ingestor/src/service/advisory/osv/mod.rs @@ -3,11 +3,12 @@ mod prefix; pub mod loader; pub mod translate; -use crate::{graph::cvss::ScoreCreator, service::Error}; +use crate::{ + graph::cvss::{ScoreCreator, ScoreInformation}, + service::Error, +}; use osv::schema::{SeverityType, Vulnerability}; -use sea_orm::{NotSet, Set}; -use trustify_entity::advisory_vulnerability_score::{self, ScoreType}; -use uuid::Uuid; +use trustify_entity::advisory_vulnerability_score::{ScoreType, Severity}; /// Load a [`Vulnerability`] from YAML, using the "classic" enum representation. pub fn from_yaml(data: &[u8]) -> Result { @@ -48,47 +49,66 @@ pub fn extract_vulnerability_ids(osv: &Vulnerability) -> impl IntoIterator for ScoreInformation { + fn from( + ( + vulnerability_id, + ScoreInfo { + r#type, + vector, + score, + severity, + }, + ): (String, ScoreInfo), + ) -> Self { + Self { + vulnerability_id, + r#type, + vector, + score, + severity, + } + } + } + // TODO: validate score type by prefix let scores = osv .severity .iter() .flatten() .flat_map(|severity| match severity.severity_type { - SeverityType::CVSSv2 => Some(( - ScoreType::V2_0, - severity.score.clone(), - 10f64, // TODO: replace with actual evaluated score - advisory_vulnerability_score::Severity::Critical, // TODO: replace with actual evaluated severity - )), - SeverityType::CVSSv3 => Some(( - match severity.score.starts_with("CVSS:3.1/") { + SeverityType::CVSSv2 => Some(ScoreInfo { + r#type: ScoreType::V2_0, + vector: severity.score.clone(), + score: 10f64, // TODO: replace with actual evaluated score + severity: Severity::Critical, // TODO: replace with actual evaluated severity + }), + SeverityType::CVSSv3 => Some(ScoreInfo { + r#type: match severity.score.starts_with("CVSS:3.1/") { true => ScoreType::V3_1, false => ScoreType::V3_0, }, - severity.score.clone(), - 10f64, // TODO: replace with actual evaluated score - advisory_vulnerability_score::Severity::Critical, // TODO: replace with actual evaluated severity - )), - SeverityType::CVSSv4 => Some(( - ScoreType::V4_0, - severity.score.clone(), - 10f64, // TODO: replace with actual evaluated score - advisory_vulnerability_score::Severity::Critical, // TODO: replace with actual evaluated severity - )), + vector: severity.score.clone(), + score: 10f64, // TODO: replace with actual evaluated score + severity: Severity::Critical, // TODO: replace with actual evaluated severity + }), + SeverityType::CVSSv4 => Some(ScoreInfo { + r#type: ScoreType::V4_0, + vector: severity.score.clone(), + score: 10f64, // TODO: replace with actual evaluated score + severity: Severity::Critical, // TODO: replace with actual evaluated severity + }), _ => None, - }) - .map( - move |(r#type, vector, score, severity)| advisory_vulnerability_score::ActiveModel { - id: Set(Uuid::now_v7()), - r#type: Set(r#type), - vector: Set(vector), - score: Set(score), - severity: Set(severity), - advisory_id: NotSet, - vulnerability_id: NotSet, - }, - ); + }); // get all vulnerability IDs @@ -98,11 +118,9 @@ pub fn extract_scores(osv: &Vulnerability, creator: &mut ScoreCreator) { // create scores for each vulnerability (alias) - creator.extend(scores.into_iter().flat_map(|score| { - ids.iter().map(move |id| { - let mut score = score.clone(); - score.vulnerability_id = Set(id.to_string()); - score - }) - })); + creator.extend( + scores + .into_iter() + .flat_map(|score| ids.iter().map(move |id| (id.to_string(), score.clone()))), + ); } From f9769f538d53eeb7ea07ec96d2631839540f8a6e Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Thu, 16 Oct 2025 14:35:23 +0200 Subject: [PATCH 21/47] chore: also use iden type for dropping --- migration/src/m0002010_example_advisory_data_migration.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migration/src/m0002010_example_advisory_data_migration.rs b/migration/src/m0002010_example_advisory_data_migration.rs index a732cf698..4ff4cbc3b 100644 --- a/migration/src/m0002010_example_advisory_data_migration.rs +++ b/migration/src/m0002010_example_advisory_data_migration.rs @@ -140,11 +140,11 @@ impl MigrationTraitWithData for Migration { .await?; manager - .drop_type(Type::drop().if_exists().name("severity").to_owned()) + .drop_type(Type::drop().if_exists().name(Severity::Table).to_owned()) .await?; manager - .drop_type(Type::drop().if_exists().name("score_type").to_owned()) + .drop_type(Type::drop().if_exists().name(ScoreType::Table).to_owned()) .await?; Ok(()) From 59cf17d807c93dc0781a6d1b8734f897ba143849 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Thu, 16 Oct 2025 17:01:51 +0200 Subject: [PATCH 22/47] chore: align column types --- migration/src/m0002010_example_advisory_data_migration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migration/src/m0002010_example_advisory_data_migration.rs b/migration/src/m0002010_example_advisory_data_migration.rs index 4ff4cbc3b..903a04d72 100644 --- a/migration/src/m0002010_example_advisory_data_migration.rs +++ b/migration/src/m0002010_example_advisory_data_migration.rs @@ -54,7 +54,7 @@ impl MigrationTraitWithData for Migration { ) .col( ColumnDef::new(AdvisoryVulnerabilityScore::VulnerabilityId) - .uuid() + .string() .not_null() .to_owned(), ) From 3aff36a957a5904b9e1c6c13b522fbeff435df59 Mon Sep 17 00:00:00 2001 From: Dejan Bosanac Date: Thu, 16 Oct 2025 13:58:07 +0200 Subject: [PATCH 23/47] impove cvss mapping logic Signed-off-by: Dejan Bosanac --- docs/book/package-lock.json | 2 +- .../src/service/advisory/cve/loader.rs | 8 +-- .../ingestor/src/service/advisory/cve/mod.rs | 69 ++++++++++++------- 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/docs/book/package-lock.json b/docs/book/package-lock.json index 5e0eee7e8..54913939c 100644 --- a/docs/book/package-lock.json +++ b/docs/book/package-lock.json @@ -1,5 +1,5 @@ { - "name": "relock-npm-lock-v2-wmxVvW", + "name": "book", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/modules/ingestor/src/service/advisory/cve/loader.rs b/modules/ingestor/src/service/advisory/cve/loader.rs index 22a09b442..91f8b2c73 100644 --- a/modules/ingestor/src/service/advisory/cve/loader.rs +++ b/modules/ingestor/src/service/advisory/cve/loader.rs @@ -98,10 +98,6 @@ impl<'g> CveLoader<'g> { .ingest_advisory(id, labels, digests, advisory_info, &tx) .await?; - let mut score_creator = ScoreCreator::new(advisory.advisory.id); - extract_scores(&cve, &mut score_creator); - score_creator.create(&tx).await?; - // Link the advisory to the backing vulnerability let advisory_vuln = advisory .link_to_vulnerability( @@ -125,6 +121,10 @@ impl<'g> CveLoader<'g> { } } + let mut score_creator = ScoreCreator::new(advisory.advisory.id); + extract_scores(&cve, &mut score_creator); + score_creator.create(&tx).await?; + // Initialize batch creator for efficient status ingestion let mut purl_status_creator = PurlStatusCreator::new(); let mut base_purls = HashSet::new(); diff --git a/modules/ingestor/src/service/advisory/cve/mod.rs b/modules/ingestor/src/service/advisory/cve/mod.rs index f49b7f074..6fd651596 100644 --- a/modules/ingestor/src/service/advisory/cve/mod.rs +++ b/modules/ingestor/src/service/advisory/cve/mod.rs @@ -1,15 +1,48 @@ use crate::graph::cvss::ScoreCreator; use cve::Cve; -use cvss::{v2_0::CvssV2, v3::CvssV3, v4_0::CvssV4}; +use cvss::{v2_0::CvssV2, v3::CvssV3, v4_0::CvssV4, Cvss}; pub mod divination; pub mod loader; +#[derive(Clone, Debug, PartialEq, Default)] +struct CvssMetric { + pub cvss_v2_0: Option, + pub cvss_v3_0: Option, + pub cvss_v3_1: Option, + pub cvss_v4_0: Option, +} + +impl From<&cve::published::Metric> for CvssMetric { + fn from(metric: &cve::published::Metric) -> Self { + Self { + cvss_v2_0: metric + .cvss_v2_0 + .as_ref() + .and_then(|v| serde_json::from_value(v.clone()).ok()), + cvss_v3_0: metric + .cvss_v3_0 + .as_ref() + .and_then(|v| serde_json::from_value(v.clone()).ok()), + cvss_v3_1: metric + .cvss_v3_1 + .as_ref() + .and_then(|v| serde_json::from_value(v.clone()).ok()), + cvss_v4_0: metric + .cvss_v4_0 + .as_ref() + .and_then(|v| serde_json::from_value(v.clone()).ok()), + } + } +} + pub fn extract_scores(cve: &Cve, creator: &mut ScoreCreator) { let Cve::Published(published) = cve else { return; }; + let vulnerability_id = &published.metadata.id; + let all_metrics = published.containers.cna.metrics.iter().chain( published .containers @@ -18,30 +51,20 @@ pub fn extract_scores(cve: &Cve, creator: &mut ScoreCreator) { .flat_map(|adp| adp.metrics.iter()), ); - let vulnerability_id = &published.metadata.id; + for cve_metric in all_metrics { + let metric = CvssMetric::from(cve_metric); - for metric in all_metrics { - if let Some(v) = &metric.cvss_v2_0 - && let Ok(score) = serde_json::from_value::(v.clone()) - { - creator.add((vulnerability_id.clone(), score)); - } - - if let Some(v) = &metric.cvss_v3_0 - && let Ok(score) = serde_json::from_value::(v.clone()) - { - creator.add((vulnerability_id.clone(), score)); - } - - if let Some(v) = &metric.cvss_v3_1 - && let Ok(score) = serde_json::from_value::(v.clone()) - { - creator.add((vulnerability_id.clone(), score)); - } + let cvss_objects: Vec = vec![ + metric.cvss_v3_1.map(Cvss::V3_1), + metric.cvss_v3_0.map(Cvss::V3_0), + metric.cvss_v2_0.map(Cvss::V2), + metric.cvss_v4_0.map(Cvss::V4), + ] + .into_iter() + .flatten() + .collect(); - if let Some(v) = &metric.cvss_v4_0 - && let Ok(score) = serde_json::from_value::(v.clone()) - { + for score in cvss_objects { creator.add((vulnerability_id.clone(), score)); } } From e99a662c4960c16fff2c59e496bbcc1be74b16f8 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Mon, 20 Oct 2025 12:14:10 +0200 Subject: [PATCH 24/47] chore: implement for csaf with v2 score --- modules/ingestor/src/service/advisory/csaf/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/ingestor/src/service/advisory/csaf/mod.rs b/modules/ingestor/src/service/advisory/csaf/mod.rs index 7aa08172e..4715aa0d0 100644 --- a/modules/ingestor/src/service/advisory/csaf/mod.rs +++ b/modules/ingestor/src/service/advisory/csaf/mod.rs @@ -19,7 +19,9 @@ pub fn extract_scores(csaf: &Csaf, creator: &mut ScoreCreator) { for score in vuln.scores.iter().flatten() { if let Some(score) = &score.cvss_v2 { - // TODO: add implementation + if let Ok(score) = serde_json::from_value::(score.clone()) { + creator.add((vulnerability_id.clone(), score)) + } } if let Some(score) = &score.cvss_v3 { From 8ea7778bb226b7f9f64f5d3b514b97ececdb2233 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Fri, 24 Oct 2025 14:53:38 +0200 Subject: [PATCH 25/47] test: pass on storage from tests to migrator --- migration/src/data/migration.rs | 24 +++++++++++++++++++++++- migration/src/data/mod.rs | 18 ++++++++++++------ migration/tests/previous.rs | 6 +++++- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index 8722a0138..6145d6d89 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -7,6 +7,7 @@ use futures::executor::block_on; use sea_orm::DbErr; use sea_orm_migration::{MigrationName, MigrationTrait, SchemaManager}; use std::{ffi::OsString, ops::Deref, sync::LazyLock}; +use tokio::task_local; use trustify_module_storage::{config::StorageConfig, service::dispatch::DispatchBackend}; pub struct MigrationWithData { @@ -18,6 +19,10 @@ pub struct MigrationWithData { static STORAGE: LazyLock = LazyLock::new(init_storage); static OPTIONS: LazyLock = LazyLock::new(init_options); +task_local! { + static TEST_STORAGE: DispatchBackend; +} + #[allow(clippy::expect_used)] fn init_storage() -> DispatchBackend { // create from env-vars only @@ -36,12 +41,28 @@ impl MigrationWithData { /// /// **NOTE:** This may panic if the storage configuration is missing. pub fn new(migration: Box) -> Self { + // if we have a test storage set, use this instead. + let storage = TEST_STORAGE + .try_with(|s| s.clone()) + .unwrap_or_else(|_| STORAGE.clone()); + Self { - storage: STORAGE.clone(), + storage, options: OPTIONS.clone(), migration, } } + + /// Set a storage backend to be used for running tests. + /// + /// This will, for the duration of the call, initialize the migrator with the provided storage + /// backend. + pub async fn run_with_test_storage(storage: impl Into, f: F) -> F::Output + where + F: Future, + { + TEST_STORAGE.scope(storage.into(), f).await + } } impl From for MigrationWithData @@ -114,6 +135,7 @@ impl MigrationTrait for MigrationWithData { &SchemaDataManager::new(manager, &self.storage, &self.options), ) .await + .inspect_err(|err| tracing::warn!("Migration failed: {err}")) } async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 60f543e9e..46d2a2c52 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -130,12 +130,18 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { .map(async |model| { let tx = db.begin().await?; - let doc = D::source(&model, storage, &tx).await.map_err(|err| { - DbErr::Migration(format!("Failed to load source document: {err}")) - })?; - f.call(doc, model, &tx).await.map_err(|err| { - DbErr::Migration(format!("Failed to process document: {err}")) - })?; + let doc = D::source(&model, storage, &tx) + .await + .inspect_err(|err| tracing::info!("Failed to load source document: {err}")) + .map_err(|err| { + DbErr::Migration(format!("Failed to load source document: {err}")) + })?; + f.call(doc, model, &tx) + .await + .inspect_err(|err| tracing::info!("Failed to process document: {err}")) + .map_err(|err| { + DbErr::Migration(format!("Failed to process document: {err}")) + })?; tx.commit().await?; diff --git a/migration/tests/previous.rs b/migration/tests/previous.rs index 543ae4aa4..24aa32e04 100644 --- a/migration/tests/previous.rs +++ b/migration/tests/previous.rs @@ -1,3 +1,4 @@ +use migration::data::MigrationWithData; use test_context::test_context; use test_log::test; use trustify_db::Database; @@ -10,7 +11,10 @@ async fn from_previous(ctx: &TrustifyMigrationContext) -> Result<(), anyhow::Err // We automatically start with a database imported from the previous commit. // But we haven't migrated to the most recent schema so far. That's done by the next step. - Database(&ctx.db).migrate().await?; + MigrationWithData::run_with_test_storage(ctx.storage.clone(), async { + Database(&ctx.db).migrate().await + }) + .await?; Ok(()) } From 73f30f3922ca95cad9da3a90e4f5dc6cb2de3d74 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Fri, 24 Oct 2025 14:53:55 +0200 Subject: [PATCH 26/47] fix: skip inserting when empty, leads to faulty SQL --- modules/ingestor/src/graph/cvss.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/ingestor/src/graph/cvss.rs b/modules/ingestor/src/graph/cvss.rs index ac782465c..cdd537051 100644 --- a/modules/ingestor/src/graph/cvss.rs +++ b/modules/ingestor/src/graph/cvss.rs @@ -156,6 +156,12 @@ impl ScoreCreator { .exec(db) .await?; + // if we have none, return now + + if scores.is_empty() { + return Ok(()); + } + // transform and set advisory let scores = scores.into_iter().map(|score| { From 983148d2c0a104aba80dee57abbb07d788650a1b Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Fri, 24 Oct 2025 16:23:35 +0200 Subject: [PATCH 27/47] refactor: rename test so drop the "example" --- migration/src/lib.rs | 4 ++-- ...sory_data_migration.rs => m0002010_add_advisory_scores.rs} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename migration/src/{m0002010_example_advisory_data_migration.rs => m0002010_add_advisory_scores.rs} (100%) diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 664a2a1ea..ad0652636 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -38,7 +38,7 @@ mod m0001180_expand_spdx_licenses_with_mappings_function; mod m0001190_optimize_product_advisory_query; mod m0001200_source_document_fk_indexes; mod m0002000_example_sbom_data_migration; -mod m0002010_example_advisory_data_migration; +mod m0002010_add_advisory_scores; pub struct Migrator; @@ -78,7 +78,7 @@ impl Migrator { .normal(m0001190_optimize_product_advisory_query::Migration) .normal(m0001200_source_document_fk_indexes::Migration) .data(m0002000_example_sbom_data_migration::Migration) - .data(m0002010_example_advisory_data_migration::Migration) + .data(m0002010_add_advisory_scores::Migration) } } diff --git a/migration/src/m0002010_example_advisory_data_migration.rs b/migration/src/m0002010_add_advisory_scores.rs similarity index 100% rename from migration/src/m0002010_example_advisory_data_migration.rs rename to migration/src/m0002010_add_advisory_scores.rs From 31fd93d54f7d6ae2174084e0ea11fe97b3d78d78 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Fri, 24 Oct 2025 16:26:53 +0200 Subject: [PATCH 28/47] chore: format --- modules/ingestor/src/service/advisory/cve/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingestor/src/service/advisory/cve/mod.rs b/modules/ingestor/src/service/advisory/cve/mod.rs index 6fd651596..b6383df77 100644 --- a/modules/ingestor/src/service/advisory/cve/mod.rs +++ b/modules/ingestor/src/service/advisory/cve/mod.rs @@ -1,6 +1,6 @@ use crate::graph::cvss::ScoreCreator; use cve::Cve; -use cvss::{v2_0::CvssV2, v3::CvssV3, v4_0::CvssV4, Cvss}; +use cvss::{Cvss, v2_0::CvssV2, v3::CvssV3, v4_0::CvssV4}; pub mod divination; pub mod loader; From c9b4f6d4a628e14475a0fc5dda4ebce9b8d5a30c Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Mon, 27 Oct 2025 10:36:36 +0100 Subject: [PATCH 29/47] test: add test for sbom --- migration/src/data/mod.rs | 6 + migration/src/lib.rs | 32 +++-- .../m0002000_example_sbom_data_migration.rs | 83 ------------ migration/tests/data.rs | 120 ++++++++++++++++++ .../ingestor/src/service/advisory/csaf/mod.rs | 8 +- 5 files changed, 150 insertions(+), 99 deletions(-) delete mode 100644 migration/src/m0002000_example_sbom_data_migration.rs create mode 100644 migration/tests/data.rs diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 46d2a2c52..1599898b5 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -227,6 +227,12 @@ impl Migrations { } } +impl Extend for Migrations { + fn extend>(&mut self, iter: T) { + self.all.extend(iter) + } +} + impl IntoIterator for Migrations { type Item = Migration; type IntoIter = std::vec::IntoIter; diff --git a/migration/src/lib.rs b/migration/src/lib.rs index ad0652636..a570ae21f 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -40,10 +40,25 @@ mod m0001200_source_document_fk_indexes; mod m0002000_example_sbom_data_migration; mod m0002010_add_advisory_scores; +pub trait MigratorExt: Send { + fn build_migrations() -> Migrations; + + fn into_migrations() -> Vec> { + // Get all migrations, wrap data migrations. This will initialize the storage config. + Self::build_migrations() + .into_iter() + .map(|migration| match migration { + Migration::Normal(migration) => migration, + Migration::Data(migration) => Box::new(MigrationWithData::new(migration)), + }) + .collect() + } +} + pub struct Migrator; -impl Migrator { - fn migrations() -> Migrations { +impl MigratorExt for Migrator { + fn build_migrations() -> Migrations { Migrations::new() .normal(m0000010_init::Migration) .normal(m0000020_add_sbom_group::Migration) @@ -82,23 +97,16 @@ impl Migrator { } } -impl MigratorWithData for Migrator { +impl MigratorWithData for M { fn data_migrations() -> Vec> { - Self::migrations().only_data() + Self::build_migrations().only_data() } } #[async_trait::async_trait] impl MigratorTrait for Migrator { fn migrations() -> Vec> { - // Get all migrations, wrap data migrations. This will initialize the storage config. - Self::migrations() - .into_iter() - .map(|migration| match migration { - Migration::Normal(migration) => migration, - Migration::Data(migration) => Box::new(MigrationWithData::new(migration)), - }) - .collect() + Self::into_migrations() } } diff --git a/migration/src/m0002000_example_sbom_data_migration.rs b/migration/src/m0002000_example_sbom_data_migration.rs deleted file mode 100644 index 603b24031..000000000 --- a/migration/src/m0002000_example_sbom_data_migration.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::{ - data::{MigrationTraitWithData, Sbom as SbomDoc, SchemaDataManager}, - sbom, -}; -use sea_orm::{ActiveModelTrait, IntoActiveModel, Set}; -use sea_orm_migration::prelude::*; -use trustify_common::advisory::cyclonedx::extract_properties_json; - -#[derive(DeriveMigrationName)] -pub struct Migration; - -#[async_trait::async_trait] -impl MigrationTraitWithData for Migration { - async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { - manager - .alter_table( - Table::alter() - .table(Sbom::Table) - .add_column_if_not_exists( - ColumnDef::new(Sbom::Properties) - .json() - .default(serde_json::Value::Null) - .to_owned(), - ) - .to_owned(), - ) - .await?; - - manager - .alter_table( - Table::alter() - .table(Sbom::Table) - .modify_column(ColumnDef::new(Sbom::Properties).not_null().to_owned()) - .to_owned(), - ) - .await?; - - manager - .process( - self, - sbom!(async |sbom, model, tx| { - let mut model = model.into_active_model(); - match sbom { - SbomDoc::CycloneDx(sbom) => { - model.properties = Set(extract_properties_json(&sbom)); - } - SbomDoc::Spdx(_sbom) => { - model.properties = Set(serde_json::Value::Object(Default::default())); - } - SbomDoc::Other(_) => { - // we ignore others - } - } - - model.save(tx).await?; - - Ok(()) - }), - ) - .await?; - - Ok(()) - } - - async fn down(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { - manager - .alter_table( - Table::alter() - .table(Sbom::Table) - .drop_column(Sbom::Properties) - .to_owned(), - ) - .await?; - - Ok(()) - } -} - -#[derive(DeriveIden)] -enum Sbom { - Table, - Properties, -} diff --git a/migration/tests/data.rs b/migration/tests/data.rs new file mode 100644 index 000000000..c5641ddec --- /dev/null +++ b/migration/tests/data.rs @@ -0,0 +1,120 @@ +use migration::{ + Migrator, MigratorExt, + data::{MigrationWithData, Migrations}, +}; +use sea_orm_migration::{MigrationTrait, MigratorTrait}; +use test_context::test_context; +use test_log::test; +use trustify_test_context::TrustifyMigrationContext; + +mod sbom { + use migration::{ + data::{MigrationTraitWithData, Sbom as SbomDoc, SchemaDataManager}, + sbom, + }; + use sea_orm::{ActiveModelTrait, IntoActiveModel, Set}; + use sea_orm_migration::prelude::*; + use trustify_common::advisory::cyclonedx::extract_properties_json; + + #[derive(DeriveMigrationName)] + pub struct Migration; + + #[async_trait::async_trait] + impl MigrationTraitWithData for Migration { + async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .add_column_if_not_exists( + ColumnDef::new(Sbom::Properties) + .json() + .default(serde_json::Value::Null) + .to_owned(), + ) + .to_owned(), + ) + .await?; + + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .modify_column(ColumnDef::new(Sbom::Properties).not_null().to_owned()) + .to_owned(), + ) + .await?; + + manager + .process( + self, + sbom!(async |sbom, model, tx| { + let mut model = model.into_active_model(); + match sbom { + SbomDoc::CycloneDx(sbom) => { + model.properties = Set(extract_properties_json(&sbom)); + } + SbomDoc::Spdx(_sbom) => { + model.properties = + Set(serde_json::Value::Object(Default::default())); + } + SbomDoc::Other(_) => { + // we ignore others + } + } + + model.save(tx).await?; + + Ok(()) + }), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .drop_column(Sbom::Properties) + .to_owned(), + ) + .await?; + + Ok(()) + } + } + + #[derive(DeriveIden)] + enum Sbom { + Table, + Properties, + } +} + +struct MigratorTest; + +impl MigratorExt for MigratorTest { + fn build_migrations() -> Migrations { + Migrator::build_migrations().data(sbom::Migration) + } +} + +impl MigratorTrait for MigratorTest { + fn migrations() -> Vec> { + Self::into_migrations() + } +} + +#[test_context(TrustifyMigrationContext)] +#[test(tokio::test)] +async fn examples(ctx: &TrustifyMigrationContext) -> Result<(), anyhow::Error> { + MigrationWithData::run_with_test_storage(ctx.storage.clone(), async { + MigratorTest::up(&ctx.db, None).await + }) + .await?; + + Ok(()) +} diff --git a/modules/ingestor/src/service/advisory/csaf/mod.rs b/modules/ingestor/src/service/advisory/csaf/mod.rs index 4715aa0d0..3bbbd820f 100644 --- a/modules/ingestor/src/service/advisory/csaf/mod.rs +++ b/modules/ingestor/src/service/advisory/csaf/mod.rs @@ -18,10 +18,10 @@ pub fn extract_scores(csaf: &Csaf, creator: &mut ScoreCreator) { }; for score in vuln.scores.iter().flatten() { - if let Some(score) = &score.cvss_v2 { - if let Ok(score) = serde_json::from_value::(score.clone()) { - creator.add((vulnerability_id.clone(), score)) - } + if let Some(score) = &score.cvss_v2 + && let Ok(score) = serde_json::from_value::(score.clone()) + { + creator.add((vulnerability_id.clone(), score)) } if let Some(score) = &score.cvss_v3 { From 83826db3ea40cd427fba1791a870530abd3e0d3d Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Mon, 27 Oct 2025 11:35:48 +0100 Subject: [PATCH 30/47] chore: bring back sbom properties --- migration/src/lib.rs | 4 +- migration/src/m0002000_add_sbom_properties.rs | 83 +++++++++++++++++++ migration/tests/data.rs | 74 +++++++++++------ 3 files changed, 134 insertions(+), 27 deletions(-) create mode 100644 migration/src/m0002000_add_sbom_properties.rs diff --git a/migration/src/lib.rs b/migration/src/lib.rs index a570ae21f..06ec89237 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -37,7 +37,7 @@ mod m0001170_non_null_source_document_id; mod m0001180_expand_spdx_licenses_with_mappings_function; mod m0001190_optimize_product_advisory_query; mod m0001200_source_document_fk_indexes; -mod m0002000_example_sbom_data_migration; +mod m0002000_add_sbom_properties; mod m0002010_add_advisory_scores; pub trait MigratorExt: Send { @@ -92,7 +92,7 @@ impl MigratorExt for Migrator { .normal(m0001180_expand_spdx_licenses_with_mappings_function::Migration) .normal(m0001190_optimize_product_advisory_query::Migration) .normal(m0001200_source_document_fk_indexes::Migration) - .data(m0002000_example_sbom_data_migration::Migration) + .data(m0002000_add_sbom_properties::Migration) .data(m0002010_add_advisory_scores::Migration) } } diff --git a/migration/src/m0002000_add_sbom_properties.rs b/migration/src/m0002000_add_sbom_properties.rs new file mode 100644 index 000000000..603b24031 --- /dev/null +++ b/migration/src/m0002000_add_sbom_properties.rs @@ -0,0 +1,83 @@ +use crate::{ + data::{MigrationTraitWithData, Sbom as SbomDoc, SchemaDataManager}, + sbom, +}; +use sea_orm::{ActiveModelTrait, IntoActiveModel, Set}; +use sea_orm_migration::prelude::*; +use trustify_common::advisory::cyclonedx::extract_properties_json; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTraitWithData for Migration { + async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .add_column_if_not_exists( + ColumnDef::new(Sbom::Properties) + .json() + .default(serde_json::Value::Null) + .to_owned(), + ) + .to_owned(), + ) + .await?; + + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .modify_column(ColumnDef::new(Sbom::Properties).not_null().to_owned()) + .to_owned(), + ) + .await?; + + manager + .process( + self, + sbom!(async |sbom, model, tx| { + let mut model = model.into_active_model(); + match sbom { + SbomDoc::CycloneDx(sbom) => { + model.properties = Set(extract_properties_json(&sbom)); + } + SbomDoc::Spdx(_sbom) => { + model.properties = Set(serde_json::Value::Object(Default::default())); + } + SbomDoc::Other(_) => { + // we ignore others + } + } + + model.save(tx).await?; + + Ok(()) + }), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .drop_column(Sbom::Properties) + .to_owned(), + ) + .await?; + + Ok(()) + } +} + +#[derive(DeriveIden)] +enum Sbom { + Table, + Properties, +} diff --git a/migration/tests/data.rs b/migration/tests/data.rs index c5641ddec..b7af616b9 100644 --- a/migration/tests/data.rs +++ b/migration/tests/data.rs @@ -2,19 +2,21 @@ use migration::{ Migrator, MigratorExt, data::{MigrationWithData, Migrations}, }; +use sea_orm::{ConnectionTrait, Statement}; use sea_orm_migration::{MigrationTrait, MigratorTrait}; use test_context::test_context; use test_log::test; use trustify_test_context::TrustifyMigrationContext; +struct MigratorTest; + mod sbom { use migration::{ + ColumnDef, DeriveIden, DeriveMigrationName, Table, async_trait, data::{MigrationTraitWithData, Sbom as SbomDoc, SchemaDataManager}, sbom, }; - use sea_orm::{ActiveModelTrait, IntoActiveModel, Set}; - use sea_orm_migration::prelude::*; - use trustify_common::advisory::cyclonedx::extract_properties_json; + use sea_orm::{ConnectionTrait, DbErr, Statement}; #[derive(DeriveMigrationName)] pub struct Migration; @@ -27,10 +29,7 @@ mod sbom { Table::alter() .table(Sbom::Table) .add_column_if_not_exists( - ColumnDef::new(Sbom::Properties) - .json() - .default(serde_json::Value::Null) - .to_owned(), + ColumnDef::new(Sbom::Foo).string().default("").to_owned(), ) .to_owned(), ) @@ -40,7 +39,7 @@ mod sbom { .alter_table( Table::alter() .table(Sbom::Table) - .modify_column(ColumnDef::new(Sbom::Properties).not_null().to_owned()) + .modify_column(ColumnDef::new(Sbom::Foo).not_null().to_owned()) .to_owned(), ) .await?; @@ -49,22 +48,24 @@ mod sbom { .process( self, sbom!(async |sbom, model, tx| { - let mut model = model.into_active_model(); - match sbom { - SbomDoc::CycloneDx(sbom) => { - model.properties = Set(extract_properties_json(&sbom)); - } - SbomDoc::Spdx(_sbom) => { - model.properties = - Set(serde_json::Value::Object(Default::default())); - } - SbomDoc::Other(_) => { - // we ignore others + // we just pick a random value + let value = match sbom { + SbomDoc::CycloneDx(sbom) => sbom.serial_number, + SbomDoc::Spdx(sbom) => { + Some(sbom.document_creation_information.spdx_document_namespace) } + SbomDoc::Other(_) => None, + }; + + if let Some(value) = value { + let stmt = Statement::from_sql_and_values( + tx.get_database_backend(), + r#"UPDATE SBOM SET FOO = $1 WHERE SBOM_ID = $2"#, + [value.into(), model.sbom_id.into()], + ); + tx.execute(stmt).await?; } - model.save(tx).await?; - Ok(()) }), ) @@ -78,7 +79,7 @@ mod sbom { .alter_table( Table::alter() .table(Sbom::Table) - .drop_column(Sbom::Properties) + .drop_column(Sbom::Foo) .to_owned(), ) .await?; @@ -90,12 +91,10 @@ mod sbom { #[derive(DeriveIden)] enum Sbom { Table, - Properties, + Foo, } } -struct MigratorTest; - impl MigratorExt for MigratorTest { fn build_migrations() -> Migrations { Migrator::build_migrations().data(sbom::Migration) @@ -116,5 +115,30 @@ async fn examples(ctx: &TrustifyMigrationContext) -> Result<(), anyhow::Error> { }) .await?; + let result = ctx + .db + .query_all(Statement::from_string( + ctx.db.get_database_backend(), + r#"SELECT FOO FROM SBOM"#, + )) + .await?; + + let foos = result + .into_iter() + .map(|row| row.try_get_by(0)) + .collect::, _>>()?; + + assert_eq!( + [ + "", + "", + "", + "https://access.redhat.com/security/data/sbom/beta/spdx/ubi8-micro-container-0ca57f3b-b0e7-4251-b32b-d2929a52f05c", + "https://access.redhat.com/security/data/sbom/beta/spdx/ubi9-container-f8098ef8-eee0-4ee6-b5d1-b00d992adef5", + "https://access.redhat.com/security/data/sbom/beta/spdx/ubi9-minimal-container-9b954617-943f-43ab-bd5b-3df62a706ed6" + ].into_iter().map(|s| s.to_owned()).collect::>(), + foos + ); + Ok(()) } From 55cd792f3efde8e22f4b814109c39ae38be24222 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Mon, 27 Oct 2025 12:02:44 +0100 Subject: [PATCH 31/47] docs: brush up a bit --- ...ments.md => 00009-re-process-documents.md} | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) rename docs/adrs/{00008-re-process-documents.md => 00009-re-process-documents.md} (72%) diff --git a/docs/adrs/00008-re-process-documents.md b/docs/adrs/00009-re-process-documents.md similarity index 72% rename from docs/adrs/00008-re-process-documents.md rename to docs/adrs/00009-re-process-documents.md index 71db27109..8c7209f9e 100644 --- a/docs/adrs/00008-re-process-documents.md +++ b/docs/adrs/00009-re-process-documents.md @@ -1,4 +1,4 @@ -# 00008. Re-process documents +# 00009. Re-process documents Date: 2025-08-08 @@ -21,14 +21,14 @@ all documents affected by this change. ### Example We do ignore all CVSS v2 scores at the moment. Adding new fields for storing v2 scores, we wouldn't have -any stored in the database without re-processing documents and extracting that information. +any stored values in the database. It therefore is necessary to re-process documents and extracting this information. ### Assumptions This ADR makes the following assumptions: * All documents are stored in the storage -* It is expected that an upgrade is actually required +* It is expected that the step of upgrading has to be performed by someone * Running such migrations is expected to take a long time * The management of infrastructure (PostgreSQL) is not in the scope of Trustify @@ -36,9 +36,10 @@ Question? Do we want to support downgrades? ## Decision -During the migration of database structures (sea orm), we also re-process all documents (if required). This would -be running during the migration job of the Helm chart and would have an impact on updates as the rollout of newer -version pods would be delayed until the migration (of data) has been finished. +During the migration of database structures (sea orm), we also re-process all documents (if required). + +For Helm deployments, this would be running during the migration job of the Helm chart and would have an impact on +updates as the rollout of newer version pods would be delayed until the migration (of data) has been finished. This would also require to prevent users from creating new documents during that time. Otherwise, we would need to re-process documents ingested during the migration time. A way of doing this could be to leverage PostgreSQL's ability @@ -72,10 +73,22 @@ error. An alternative to this could also be to configure the system first to go into "read-only mode", by using a default transaction mode of read-only. +## Consequences + +Migrations which do re-process data have to be written in a way, that they can be run and re-run without failing +during the migration of the schema (e.g. add "if not exist"). In this case, the data migration job can be run +"out of band" (beforehand) and the data be processed. Then, the actual upgrade and schema migration can run. + + ## Open items +* [ ] Do we want to support downgrades? + * I'd say no. Downgrades could also be handled by keeping a snapshot of the original database. * [ ] How to handle unparsable or failing documents during migration? + * Pass them in as "unsupported" * [ ] Add a version number to the document, tracking upgrades + * This adds some complexity, but might allow to track the progress and identify upgraded documents. This could also + ensure the correct order of applying data migrations out of band. ## Alternative approaches @@ -84,9 +97,9 @@ transaction mode of read-only. We create a similar module as for the importer. Running migrations after an upgrade. Accepting that in the meantime, we might service inaccurate data. -* 👎 Might serve inaccurate data for a while for a longer time +* 👎 Might serve inaccurate data for a longer time * 👎 Can't fully migrate database (new mandatory field won't work) -* 👍 Upgrade process is faster and less complex +* 👍 Upgrade process itself is faster and less complex * 👎 Requires some coordination between instances (only one processor at a time, maybe one after the other) ### Option 3 @@ -101,6 +114,16 @@ original sources. * 👎 Won't work for manual (API) uploads * 👎 Would require removing optimizations for existing documents +### Option 4 + +Have the operator orchestrate the process of switching the database into read-only mode and running the migrations. + +* 👍 Very user friendly +* 👎 Rather complex +* 👎 Required access to the user's DB infrastructure + +This adds a lot of user-friendliness. However, it also is rather complex and so we should, as a first step, have this +as a manual step. ## Consequences From ca2cc3c1d21deddbe84878eacf2dc06eaf0b48f2 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Tue, 28 Oct 2025 13:40:23 +0100 Subject: [PATCH 32/47] test: sort order, to make test stable --- migration/tests/data.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/migration/tests/data.rs b/migration/tests/data.rs index b7af616b9..21f050c82 100644 --- a/migration/tests/data.rs +++ b/migration/tests/data.rs @@ -4,6 +4,7 @@ use migration::{ }; use sea_orm::{ConnectionTrait, Statement}; use sea_orm_migration::{MigrationTrait, MigratorTrait}; +use std::collections::BTreeSet; use test_context::test_context; use test_log::test; use trustify_test_context::TrustifyMigrationContext; @@ -126,7 +127,7 @@ async fn examples(ctx: &TrustifyMigrationContext) -> Result<(), anyhow::Error> { let foos = result .into_iter() .map(|row| row.try_get_by(0)) - .collect::, _>>()?; + .collect::, _>>()?; assert_eq!( [ @@ -136,7 +137,7 @@ async fn examples(ctx: &TrustifyMigrationContext) -> Result<(), anyhow::Error> { "https://access.redhat.com/security/data/sbom/beta/spdx/ubi8-micro-container-0ca57f3b-b0e7-4251-b32b-d2929a52f05c", "https://access.redhat.com/security/data/sbom/beta/spdx/ubi9-container-f8098ef8-eee0-4ee6-b5d1-b00d992adef5", "https://access.redhat.com/security/data/sbom/beta/spdx/ubi9-minimal-container-9b954617-943f-43ab-bd5b-3df62a706ed6" - ].into_iter().map(|s| s.to_owned()).collect::>(), + ].into_iter().map(|s| s.to_owned()).collect::>(), foos ); From 6159ee93b0580e3693d10d66f1b8def344936b9c Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Tue, 28 Oct 2025 16:26:47 +0100 Subject: [PATCH 33/47] chore: add additional SBOM column --- .../src/vulnerability/model/details/vulnerability_advisory.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/fundamental/src/vulnerability/model/details/vulnerability_advisory.rs b/modules/fundamental/src/vulnerability/model/details/vulnerability_advisory.rs index 046ee07fd..8c92fbb59 100644 --- a/modules/fundamental/src/vulnerability/model/details/vulnerability_advisory.rs +++ b/modules/fundamental/src/vulnerability/model/details/vulnerability_advisory.rs @@ -215,6 +215,7 @@ impl VulnerabilityAdvisorySummary { "sbom"."data_licenses" AS "sbom$data_licenses", "sbom"."source_document_id" AS "sbom$source_document_id", "sbom"."labels" AS "sbom$labels", + "sbom"."properties" AS "sbom$properties", "sbom_package"."sbom_id" AS "sbom_package$sbom_id", "sbom_package"."node_id" AS "sbom_package$node_id", "sbom_package"."version" AS "sbom_package$version", From 8a78ff7a9643ecc05034df67a7d47a0d20b5b820 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 29 Oct 2025 10:09:56 +0100 Subject: [PATCH 34/47] chore: update example --- data-migration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-migration.yaml b/data-migration.yaml index 77f610d36..bfd6f3615 100644 --- a/data-migration.yaml +++ b/data-migration.yaml @@ -25,7 +25,7 @@ spec: - /usr/local/bin/trustd - db - data - - m0002000_example_data_migration # name of the migration + - m0002010_add_advisory_scores # name of the migration env: - name: MIGRATION_DATA_CONCURRENT value: "5" # in-process parallelism From 679707782d36c4927a3170847854e3ed61950220 Mon Sep 17 00:00:00 2001 From: Dejan Bosanac Date: Wed, 29 Oct 2025 13:20:56 +0100 Subject: [PATCH 35/47] chore: adapt CSAF CVSS v3 handling to use JSON deserialization Update csaf-rs dependency to use the "cvss" library where CVSS scores are stored as raw JSON values instead of pre-parsed objects. Assisted-By: Claude Signed-off-by: Dejan Bosanac --- Cargo.lock | 6 +---- .../src/service/advisory/csaf/loader.rs | 26 ++++++++++++------- .../ingestor/src/service/advisory/csaf/mod.rs | 19 +++++--------- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 297daa5ee..bf42692b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2090,11 +2090,10 @@ dependencies = [ [[package]] name = "csaf" version = "0.5.0" -source = "git+https://github.com/trustification/csaf-rs#17620a225744b4a18845d4f7bf63354e01109b91" +source = "git+https://github.com/trustification/csaf-rs?branch=cvss#d6df319076cdc685a2cded0cb6797ad7ce92b9c9" dependencies = [ "chrono", "cpe", - "cvss 2.1.1", "packageurl", "serde", "serde_json", @@ -2214,9 +2213,6 @@ name = "cvss" version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4f643e062e9a8e26edea270945e05011c441ca6a56e9d9d4464c6b0be1352bd" -dependencies = [ - "serde", -] [[package]] name = "darling" diff --git a/modules/ingestor/src/service/advisory/csaf/loader.rs b/modules/ingestor/src/service/advisory/csaf/loader.rs index 0fd645bde..d5d152b44 100644 --- a/modules/ingestor/src/service/advisory/csaf/loader.rs +++ b/modules/ingestor/src/service/advisory/csaf/loader.rs @@ -19,6 +19,7 @@ use csaf::{ Csaf, vulnerability::{ProductStatus, Vulnerability}, }; +use cvss::v3::CvssV3; use hex::ToHex; use sbom_walker::report::ReportSink; use sea_orm::{ConnectionTrait, TransactionTrait}; @@ -187,16 +188,23 @@ impl<'g> CsafLoader<'g> { } for score in vulnerability.scores.iter().flatten() { - if let Some(v3) = &score.cvss_v3 { - match Cvss3Base::from_str(&v3.to_string()) { - Ok(cvss3) => { - log::debug!("{cvss3:?}"); - advisory_vulnerability - .ingest_cvss3_score(cvss3, connection) - .await?; - } + if let Some(cvss_v3) = &score.cvss_v3 { + match serde_json::from_value::(cvss_v3.clone()) { + Ok(cvss) => match Cvss3Base::from_str(&cvss.vector_string) { + Ok(cvss3) => { + log::debug!("{cvss3:?}"); + advisory_vulnerability + .ingest_cvss3_score(cvss3, connection) + .await?; + } + Err(err) => { + let msg = format!("Unable to parse CVSS3: {err:#?}"); + log::info!("{msg}"); + report.error(msg); + } + }, Err(err) => { - let msg = format!("Unable to parse CVSS3: {err:#?}"); + let msg = format!("Unable to deserialize CVSS3 JSON: {err:#?}"); log::info!("{msg}"); report.error(msg); } diff --git a/modules/ingestor/src/service/advisory/csaf/mod.rs b/modules/ingestor/src/service/advisory/csaf/mod.rs index 3bbbd820f..96501d50c 100644 --- a/modules/ingestor/src/service/advisory/csaf/mod.rs +++ b/modules/ingestor/src/service/advisory/csaf/mod.rs @@ -5,9 +5,9 @@ mod util; mod creator; pub use creator::*; -use crate::graph::cvss::{ScoreCreator, ScoreInformation}; +use crate::graph::cvss::ScoreCreator; use csaf::Csaf; -use trustify_entity::advisory_vulnerability_score::ScoreType; +use cvss::v3::CvssV3; /// Extract scores from a CSAF document pub fn extract_scores(csaf: &Csaf, creator: &mut ScoreCreator) { @@ -24,17 +24,10 @@ pub fn extract_scores(csaf: &Csaf, creator: &mut ScoreCreator) { creator.add((vulnerability_id.clone(), score)) } - if let Some(score) = &score.cvss_v3 { - // TODO: maybe use raw values from JSON - let vector = score.to_string(); - let score = score.score(); - creator.add(ScoreInformation { - vulnerability_id: vulnerability_id.clone(), - r#type: ScoreType::V3_0, - vector, - score: score.value(), - severity: score.severity().into(), - }) + if let Some(cvss_v3) = &score.cvss_v3 + && let Ok(cvss) = serde_json::from_value::(cvss_v3.clone()) + { + creator.add((vulnerability_id.clone(), cvss)) } } } From f0783d691e9f589601b5324bd5f66c81ea80b53e Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 29 Oct 2025 15:26:08 +0100 Subject: [PATCH 36/47] chore: only create the type if it doesn't exist --- migration/src/m0002010_add_advisory_scores.rs | 70 +++++++++++++------ 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/migration/src/m0002010_add_advisory_scores.rs b/migration/src/m0002010_add_advisory_scores.rs index 903a04d72..d76ef983b 100644 --- a/migration/src/m0002010_add_advisory_scores.rs +++ b/migration/src/m0002010_add_advisory_scores.rs @@ -2,7 +2,7 @@ use crate::{ advisory, data::{Advisory as AdvisoryDoc, MigrationTraitWithData, SchemaDataManager}, }; -use sea_orm::sea_query::extension::postgres::Type; +use sea_orm::sea_query::extension::postgres::*; use sea_orm_migration::prelude::*; use strum::VariantNames; use trustify_module_ingestor::{ @@ -13,26 +13,56 @@ use trustify_module_ingestor::{ #[derive(DeriveMigrationName)] pub struct Migration; +/// create a type, if it not already exists +/// +/// This is required as Postgres doesn't support `CREATE TYPE IF NOT EXISTS` +pub async fn create_enum_if_not_exists( + manager: &SchemaManager<'_>, + name: impl IntoIden + Clone, + values: I, +) -> Result<(), DbErr> +where + T: IntoIden, + I: IntoIterator, +{ + let builder = manager.get_connection().get_database_backend(); + let r#type = name.clone().into_iden(); + let stmt = builder.build(Type::create().as_enum(name).values(values)); + let stmt = format!( + r#" +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type WHERE typname = '{name}' + ) THEN + {stmt}; + END IF; +END$$; +"#, + name = r#type.to_string() + ); + + manager.get_connection().execute_unprepared(&stmt).await?; + + Ok(()) +} + #[async_trait::async_trait] impl MigrationTraitWithData for Migration { async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { - manager - .create_type( - Type::create() - .as_enum(Severity::Table) - .values(Severity::VARIANTS.iter().skip(1).copied()) - .to_owned(), - ) - .await?; - - manager - .create_type( - Type::create() - .as_enum(ScoreType::Table) - .values(ScoreType::VARIANTS.iter().skip(1).copied()) - .to_owned(), - ) - .await?; + create_enum_if_not_exists( + manager, + Severity::Table, + Severity::VARIANTS.iter().skip(1).copied(), + ) + .await?; + + create_enum_if_not_exists( + manager, + ScoreType::Table, + ScoreType::VARIANTS.iter().skip(1).copied(), + ) + .await?; manager .create_table( @@ -170,7 +200,7 @@ enum AdvisoryVulnerabilityScore { Severity, } -#[derive(DeriveIden, strum::VariantNames, strum::Display)] +#[derive(DeriveIden, strum::VariantNames, strum::Display, Clone)] #[allow(unused)] enum ScoreType { Table, @@ -184,7 +214,7 @@ enum ScoreType { V4_0, } -#[derive(DeriveIden, strum::VariantNames, strum::Display)] +#[derive(DeriveIden, strum::VariantNames, strum::Display, Clone)] #[strum(serialize_all = "lowercase")] #[allow(unused)] enum Severity { From 92339a27e3b9e75aeeef289d37ce80add81da52f Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 29 Oct 2025 16:29:54 +0100 Subject: [PATCH 37/47] test: the cvss score parsing should no longer fail --- modules/ingestor/tests/issues.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/ingestor/tests/issues.rs b/modules/ingestor/tests/issues.rs index a52907aa3..af940e67c 100644 --- a/modules/ingestor/tests/issues.rs +++ b/modules/ingestor/tests/issues.rs @@ -6,7 +6,7 @@ use trustify_common::id::Id; use trustify_test_context::TrustifyContext; #[test_context(TrustifyContext)] -#[test(actix_web::test)] +#[test(tokio::test)] /// Ingested SBOM should not fail async fn issue_1492(ctx: &TrustifyContext) -> anyhow::Result<()> { let result = ctx @@ -19,17 +19,14 @@ async fn issue_1492(ctx: &TrustifyContext) -> anyhow::Result<()> { } #[test_context(TrustifyContext)] -#[test(actix_web::test)] +#[test(tokio::test)] /// Ingested SBOM should not fail async fn cvss_issue_1(ctx: &TrustifyContext) -> anyhow::Result<()> { let result = ctx .ingest_document("csaf/issues/cvss_1/ssa-054046.json") - .await; + .await?; - assert_eq!( - result.expect_err("must be an error").to_string(), - "unknown CVSS metric name: `E` at line 3001 column 11" - ); + assert!(matches!(result.id, Id::Uuid(_))); Ok(()) } From 70c0b4b1930512a9d8b9b63e9735a016d220e9ef Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Thu, 30 Oct 2025 14:20:42 +0100 Subject: [PATCH 38/47] refactor: pull out check if the migration should be ignored --- migration/src/data/migration.rs | 8 +------- migration/src/data/mod.rs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index 6145d6d89..f6f92d8d9 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -107,13 +107,7 @@ impl<'c> SchemaDataManager<'c> { D: Document, N: MigrationName + Send + Sync, { - if self.options.skip_all { - // we skip all migration - return Ok(()); - } - - if self.options.skip.iter().any(|s| s == name.name()) { - // we skip a list of migrations, and it's on the list + if self.options.should_skip(name.name()) { return Ok(()); } diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 1599898b5..522376795 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -73,6 +73,22 @@ impl Default for Options { } } +impl Options { + pub fn should_skip(&self, name: &str) -> bool { + if self.skip_all { + // we skip all migration + return true; + } + + if self.skip.iter().any(|s| s == name) { + // we skip a list of migrations, and it's on the list + return true; + } + + false + } +} + impl From<&Options> for Partition { fn from(value: &Options) -> Self { Self { From 5fb0a8fe884efb585713e8afdbf7f311cedee240 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Thu, 30 Oct 2025 14:37:03 +0100 Subject: [PATCH 39/47] docs: document the test case a bit --- migration/tests/data.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/migration/tests/data.rs b/migration/tests/data.rs index 21f050c82..a0c5f9b91 100644 --- a/migration/tests/data.rs +++ b/migration/tests/data.rs @@ -108,6 +108,11 @@ impl MigratorTrait for MigratorTest { } } +/// test an example migration base on an existing database dump from the previous commit. +/// +/// The idea is to add a new field and populate it with data. +/// +/// As we don't actually change the entities, this has to work with plain SQL. #[test_context(TrustifyMigrationContext)] #[test(tokio::test)] async fn examples(ctx: &TrustifyMigrationContext) -> Result<(), anyhow::Error> { From f5c5bb071d2ddb2b6d8d7a7255ccf60e2e2904f1 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Fri, 31 Oct 2025 09:27:13 +0100 Subject: [PATCH 40/47] test: add a test for re-running m0002010 --- common/src/db/mod.rs | 4 ++ migration/src/bin/data.rs | 12 +++--- migration/src/data/migration.rs | 19 ++++++++-- migration/src/data/mod.rs | 6 +++ migration/src/data/run.rs | 30 +++++++++------ migration/tests/data/m0002010.rs | 44 ++++++++++++++++++++++ migration/tests/{data.rs => data/main.rs} | 4 +- migration/tests/previous.rs | 2 +- test-context/src/ctx/migration.rs | 45 ++++++++++++++++++++--- test-context/src/migration.rs | 11 +++--- trustd/src/db.rs | 8 ++-- 11 files changed, 149 insertions(+), 36 deletions(-) create mode 100644 migration/tests/data/m0002010.rs rename migration/tests/{data.rs => data/main.rs} (98%) diff --git a/common/src/db/mod.rs b/common/src/db/mod.rs index c68f8dece..aadfdc8ad 100644 --- a/common/src/db/mod.rs +++ b/common/src/db/mod.rs @@ -103,6 +103,10 @@ impl Database { pub fn name(&self) -> &str { &self.name } + + pub fn into_connection(self) -> DatabaseConnection { + self.db + } } impl Deref for Database { diff --git a/migration/src/bin/data.rs b/migration/src/bin/data.rs index 70bf2265d..f67a6b8c6 100644 --- a/migration/src/bin/data.rs +++ b/migration/src/bin/data.rs @@ -1,7 +1,7 @@ use clap::Parser; use migration::{ Migrator, - data::{Direction, MigratorWithData, Options, Runner}, + data::{Database, Direction, MigratorWithData, Options, Runner}, }; use trustify_module_storage::config::StorageConfig; @@ -86,10 +86,12 @@ impl Run { direction, storage, migrations: self.migrations, - database_url: self - .database_url - .expect("Environment variable 'DATABASE_URL' not set"), - database_schema: self.database_schema, + database: Database::Config { + url: self + .database_url + .expect("Environment variable 'DATABASE_URL' not set"), + schema: self.database_schema, + }, options: self.options, } .run::() diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index f6f92d8d9..b00383de5 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -21,6 +21,7 @@ static OPTIONS: LazyLock = LazyLock::new(init_options); task_local! { static TEST_STORAGE: DispatchBackend; + static TEST_OPTIONS: Options; } #[allow(clippy::expect_used)] @@ -46,9 +47,13 @@ impl MigrationWithData { .try_with(|s| s.clone()) .unwrap_or_else(|_| STORAGE.clone()); + let options = TEST_OPTIONS + .try_with(|o| o.clone()) + .unwrap_or_else(|_| OPTIONS.clone()); + Self { storage, - options: OPTIONS.clone(), + options, migration, } } @@ -57,11 +62,19 @@ impl MigrationWithData { /// /// This will, for the duration of the call, initialize the migrator with the provided storage /// backend. - pub async fn run_with_test_storage(storage: impl Into, f: F) -> F::Output + pub async fn run_with_test( + storage: impl Into, + options: impl Into, + f: F, + ) -> F::Output where F: Future, { - TEST_STORAGE.scope(storage.into(), f).await + TEST_STORAGE + .scope(storage.into(), async { + TEST_OPTIONS.scope(options.into(), f).await + }) + .await } } diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 522376795..24ead6038 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -73,6 +73,12 @@ impl Default for Options { } } +impl From<()> for Options { + fn from(_: ()) -> Self { + Self::default() + } +} + impl Options { pub fn should_skip(&self, name: &str) -> bool { if self.skip_all { diff --git a/migration/src/data/run.rs b/migration/src/data/run.rs index e73e55e01..b576870fa 100644 --- a/migration/src/data/run.rs +++ b/migration/src/data/run.rs @@ -1,9 +1,8 @@ use crate::data::{MigratorWithData, Options, SchemaDataManager}; use anyhow::bail; -use sea_orm::{ConnectOptions, Database}; +use sea_orm::ConnectOptions; use sea_orm_migration::{IntoSchemaManagerConnection, SchemaManager}; -use std::collections::HashMap; -use std::time::SystemTime; +use std::{collections::HashMap, time::SystemTime}; use trustify_module_storage::service::dispatch::DispatchBackend; #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] @@ -14,14 +13,18 @@ pub enum Direction { } pub struct Runner { - pub database_url: String, - pub database_schema: Option, + pub database: Database, pub storage: DispatchBackend, pub direction: Direction, pub migrations: Vec, pub options: Options, } +pub enum Database { + Config { url: String, schema: Option }, + Provided(sea_orm::DatabaseConnection), +} + impl Runner { pub async fn run(self) -> anyhow::Result<()> { let migrations = M::data_migrations() @@ -38,15 +41,20 @@ impl Runner { running.push(migration); } - let schema = self.database_schema.unwrap_or_else(|| "public".to_owned()); + let database = match self.database { + Database::Config { url, schema } => { + let schema = schema.unwrap_or_else(|| "public".to_owned()); - let connect_options = ConnectOptions::new(self.database_url) - .set_schema_search_path(schema) - .to_owned(); + let connect_options = ConnectOptions::new(url) + .set_schema_search_path(schema) + .to_owned(); - let db = Database::connect(connect_options).await?; + sea_orm::Database::connect(connect_options).await? + } + Database::Provided(database) => database, + }; - let manager = SchemaManager::new(db.into_schema_manager_connection()); + let manager = SchemaManager::new(database.into_schema_manager_connection()); let manager = SchemaDataManager::new(&manager, &self.storage, &self.options); for run in running { diff --git a/migration/tests/data/m0002010.rs b/migration/tests/data/m0002010.rs new file mode 100644 index 000000000..4fdefb4f2 --- /dev/null +++ b/migration/tests/data/m0002010.rs @@ -0,0 +1,44 @@ +use crate::MigratorTest; +use migration::Migrator; +use migration::data::{Database, Direction, MigrationWithData, Options, Runner}; +use sea_orm_migration::MigratorTrait; +use test_context::test_context; +use test_log::test; +use trustify_test_context::{TrustifyMigrationContext, commit, ctx::DumpId}; + +commit!(Commit("8c6ad23172e66a6c923dcc8f702e6125a8d48723")); + +#[test_context(TrustifyMigrationContext)] +#[test(tokio::test)] +async fn examples( + ctx: &TrustifyMigrationContext, /* commit previous to this PR */ +) -> Result<(), anyhow::Error> { + let migrations = vec!["m0002010_add_advisory_scores".into()]; + + // first run the data migration + Runner { + direction: Direction::Up, + storage: ctx.storage.clone().into(), + migrations: migrations.clone(), + database: Database::Provided(ctx.db.clone().into_connection()), + options: Default::default(), + } + .run::() + .await?; + + // now run the migrations, but skip the already run migration + + MigrationWithData::run_with_test( + ctx.storage.clone(), + Options { + skip: migrations, + ..Default::default() + }, + async { MigratorTest::up(&ctx.db, None).await }, + ) + .await?; + + // done + + Ok(()) +} diff --git a/migration/tests/data.rs b/migration/tests/data/main.rs similarity index 98% rename from migration/tests/data.rs rename to migration/tests/data/main.rs index a0c5f9b91..c97f9871a 100644 --- a/migration/tests/data.rs +++ b/migration/tests/data/main.rs @@ -1,3 +1,5 @@ +mod m0002010; + use migration::{ Migrator, MigratorExt, data::{MigrationWithData, Migrations}, @@ -116,7 +118,7 @@ impl MigratorTrait for MigratorTest { #[test_context(TrustifyMigrationContext)] #[test(tokio::test)] async fn examples(ctx: &TrustifyMigrationContext) -> Result<(), anyhow::Error> { - MigrationWithData::run_with_test_storage(ctx.storage.clone(), async { + MigrationWithData::run_with_test(ctx.storage.clone(), (), async { MigratorTest::up(&ctx.db, None).await }) .await?; diff --git a/migration/tests/previous.rs b/migration/tests/previous.rs index 24aa32e04..7035707d0 100644 --- a/migration/tests/previous.rs +++ b/migration/tests/previous.rs @@ -11,7 +11,7 @@ async fn from_previous(ctx: &TrustifyMigrationContext) -> Result<(), anyhow::Err // We automatically start with a database imported from the previous commit. // But we haven't migrated to the most recent schema so far. That's done by the next step. - MigrationWithData::run_with_test_storage(ctx.storage.clone(), async { + MigrationWithData::run_with_test(ctx.storage.clone(), (), async { Database(&ctx.db).migrate().await }) .await?; diff --git a/test-context/src/ctx/migration.rs b/test-context/src/ctx/migration.rs index 5372fb9b5..93e309715 100644 --- a/test-context/src/ctx/migration.rs +++ b/test-context/src/ctx/migration.rs @@ -1,15 +1,43 @@ use crate::{TrustifyTestContext, migration::Migration}; use anyhow::Context; +use std::borrow::Cow; +use std::marker::PhantomData; use std::ops::Deref; use tar::Archive; use test_context::AsyncTestContext; use trustify_db::embedded::{Options, Source, default_settings}; use trustify_module_storage::service::fs::FileSystemBackend; +#[macro_export] +macro_rules! commit { + ($t:ident($id:literal)) => { + pub struct $t; + + impl DumpId for $t { + fn dump_id() -> Option<&'static str> { + Some($id) + } + } + }; +} + +pub trait DumpId { + fn dump_id() -> Option<&'static str>; +} + +impl DumpId for () { + fn dump_id() -> Option<&'static str> { + None + } +} + /// Creates a database and imports the previous DB and storage dump. -pub struct TrustifyMigrationContext(pub(crate) TrustifyTestContext); +pub struct TrustifyMigrationContext( + pub(crate) TrustifyTestContext, + PhantomData, +); -impl Deref for TrustifyMigrationContext { +impl Deref for TrustifyMigrationContext { type Target = TrustifyTestContext; fn deref(&self) -> &Self::Target { @@ -17,10 +45,14 @@ impl Deref for TrustifyMigrationContext { } } -impl TrustifyMigrationContext { +impl TrustifyMigrationContext { pub async fn new() -> anyhow::Result { let migration = Migration::new().expect("failed to create migration manager"); - let base = migration.provide().await?; + let id: Cow<'static, str> = match ID::dump_id() { + Some(id) => format!("commit-{id}").into(), + None => "latest".into(), + }; + let base = migration.provide(&id).await?; // create storage @@ -50,13 +82,14 @@ impl TrustifyMigrationContext { Ok(Self( TrustifyTestContext::new(db, storage, tmp, postgresql).await, + Default::default(), )) } } -impl AsyncTestContext for TrustifyMigrationContext { +impl AsyncTestContext for TrustifyMigrationContext { async fn setup() -> Self { - TrustifyMigrationContext::new() + Self::new() .await .expect("failed to create migration context") } diff --git a/test-context/src/migration.rs b/test-context/src/migration.rs index 052e046f5..bd2ae3734 100644 --- a/test-context/src/migration.rs +++ b/test-context/src/migration.rs @@ -68,8 +68,8 @@ impl Migration { /// Provide the base dump path, for this branch. /// /// This may include downloading content from S3. - pub async fn provide(&self) -> anyhow::Result { - let base = self.base.join(&self.branch); + pub async fn provide(&self, id: &str) -> anyhow::Result { + let base = self.base.join(&self.branch).join(id); log::info!("branch base path: '{}'", base.display()); @@ -101,6 +101,7 @@ impl Migration { &self.bucket, &self.region, &self.branch, + id, files, ) .await? @@ -297,6 +298,7 @@ async fn download_artifacts( bucket: &str, region: &str, branch: &str, + commit: &str, files: impl IntoIterator>, ) -> anyhow::Result<()> { let base = base.as_ref(); @@ -305,10 +307,7 @@ async fn download_artifacts( let file = file.as_ref(); vec![file.to_string(), format!("{file}.sha256")] }) { - let url = format!( - "https://{}.s3.{}.amazonaws.com/{}/latest/{}", - bucket, region, branch, file - ); + let url = format!("https://{bucket}.s3.{region}.amazonaws.com/{branch}/{commit}/{file}",); log::info!("downloading file: '{url}'"); diff --git a/trustd/src/db.rs b/trustd/src/db.rs index bb0d61da2..32ab6a2b1 100644 --- a/trustd/src/db.rs +++ b/trustd/src/db.rs @@ -1,4 +1,4 @@ -use migration::data::{Direction, Options, Runner}; +use migration::data::{self, Direction, Options, Runner}; use postgresql_embedded::{PostgreSQL, VersionReq}; use std::{collections::HashMap, env, fs::create_dir_all, process::ExitCode, time::Duration}; use trustify_common::{config::Database, db}; @@ -134,8 +134,10 @@ impl Data { Ok(db) => { trustify_db::Database(&db) .data_migrate(Runner { - database_url: database.to_url(), - database_schema: None, + database: data::Database::Config { + url: database.to_url(), + schema: None, + }, storage: storage.into_storage(false).await?, direction, migrations, From b335a12d9b1580a723e385834122a1145e946725 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Mon, 10 Nov 2025 12:00:11 +0100 Subject: [PATCH 41/47] docs: improve on documentation --- ...ments.md => 00011-re-process-documents.md} | 21 +++++---- migration/src/data/document/mod.rs | 1 + migration/src/data/migration.rs | 2 + migration/src/data/mod.rs | 44 +++++++++++++++++++ migration/src/data/partition.rs | 4 ++ 5 files changed, 61 insertions(+), 11 deletions(-) rename docs/adrs/{00009-re-process-documents.md => 00011-re-process-documents.md} (94%) diff --git a/docs/adrs/00009-re-process-documents.md b/docs/adrs/00011-re-process-documents.md similarity index 94% rename from docs/adrs/00009-re-process-documents.md rename to docs/adrs/00011-re-process-documents.md index 8c7209f9e..d77a102e3 100644 --- a/docs/adrs/00009-re-process-documents.md +++ b/docs/adrs/00011-re-process-documents.md @@ -1,4 +1,4 @@ -# 00009. Re-process documents +# 00011. Re-process documents Date: 2025-08-08 @@ -28,7 +28,7 @@ any stored values in the database. It therefore is necessary to re-process docum This ADR makes the following assumptions: * All documents are stored in the storage -* It is expected that the step of upgrading has to be performed by someone +* It is expected that the step of upgrading has to be performed by someone, it is not magically happening * Running such migrations is expected to take a long time * The management of infrastructure (PostgreSQL) is not in the scope of Trustify @@ -52,7 +52,7 @@ We could provide an endpoint to the UI, reporting the fact that the system is in * 👍 Can fully migrate database (create mandatory field as optional -> re-process -> make mandatory) * 👍 Might allow for an out-of-band migration of data, before running the upgrade (even on a staging env) * 👍 Would allow to continue serving data while the process is running -* 👎 Might be tricky to create a combined re-processing of multiple ones +* 👎 Might be tricky to create a combined re-processing of multiple ones at the same time * 👎 Might block an upgrade if re-processing fails We do want to support different approaches of this migration. Depending on the needs of the user, the size of the @@ -77,8 +77,14 @@ transaction mode of read-only. Migrations which do re-process data have to be written in a way, that they can be run and re-run without failing during the migration of the schema (e.g. add "if not exist"). In this case, the data migration job can be run -"out of band" (beforehand) and the data be processed. Then, the actual upgrade and schema migration can run. +"out of band" (beforehand) and the data be processed. Then, the actual upgrade and schema migration can run, keeping +the SeaORM process. +* The migration will block the upgrade process until it is finished +* Ansible and the operator will need to handle this as well +* The system will become read-only during a migration +* The UI should let the user know the system is in read-only mode. This is a feature which has to be rolled out before + the data migration can be used. ## Open items @@ -124,10 +130,3 @@ Have the operator orchestrate the process of switching the database into read-on This adds a lot of user-friendliness. However, it also is rather complex and so we should, as a first step, have this as a manual step. - -## Consequences - -* The migration will block the upgrade process until it is finished -* Ansible and the operator will need to handle this as well -* The system will become read-only during a migration -* The UI needs to provide a page for monitoring the migration state. The backend needs to provide appropriate APIs. diff --git a/migration/src/data/document/mod.rs b/migration/src/data/document/mod.rs index de3491077..440548d78 100644 --- a/migration/src/data/document/mod.rs +++ b/migration/src/data/document/mod.rs @@ -14,6 +14,7 @@ use trustify_entity::source_document; use trustify_module_storage::service::{StorageBackend, StorageKey}; use uuid::Uuid; +/// A document eligible for re-processing. #[allow(async_fn_in_trait)] pub trait Document: Sized + Send + Sync { type Model: Partitionable + Send; diff --git a/migration/src/data/migration.rs b/migration/src/data/migration.rs index b00383de5..ccc1bb6bb 100644 --- a/migration/src/data/migration.rs +++ b/migration/src/data/migration.rs @@ -10,6 +10,7 @@ use std::{ffi::OsString, ops::Deref, sync::LazyLock}; use tokio::task_local; use trustify_module_storage::{config::StorageConfig, service::dispatch::DispatchBackend}; +/// A migration which also processes data. pub struct MigrationWithData { pub storage: DispatchBackend, pub options: Options, @@ -87,6 +88,7 @@ where } } +/// A [`SchemaManager`], extended with data migration features. pub struct SchemaDataManager<'c> { pub manager: &'c SchemaManager<'c>, storage: &'c DispatchBackend, diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 24ead6038..8ee86eb0c 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -80,6 +80,10 @@ impl From<()> for Options { } impl Options { + /// Check if we should skip a data migration. Returns `true` if it should be skipped. + /// + /// Skipping means that the "data" part of the migration should not be processes. The schema + /// part still will be processes. pub fn should_skip(&self, name: &str) -> bool { if self.skip_all { // we skip all migration @@ -104,6 +108,7 @@ impl From<&Options> for Partition { } } +/// A trait for processing documents pub trait DocumentProcessor { fn process( &self, @@ -116,6 +121,42 @@ pub trait DocumentProcessor { } impl<'c> DocumentProcessor for SchemaManager<'c> { + /// Process documents for a schema *data* migration. + /// + /// ## Pre-requisites + /// + /// The database should be maintenance mode. Meaning that the actual application should be + /// running from a read-only clone for the time of processing. + /// + /// ## Partitioning + /// + /// This will partition documents and only process documents selected for *this* partition. + /// The partition configuration normally comes from outside, as configuration through env-vars. + /// + /// This means that there may be other instances of this processor running in a different + /// process instance. However, not touching documents of our partition. + /// + /// ## Transaction strategy + /// + /// The processor will identify all documents, filtering out all which are not part of this + /// partition. This is done in a dedicated transaction. As the database is supposed to be in + /// read-only mode for the running instance, this is ok as no additional documents will be + /// created during the time of processing. + /// + /// Next, it is processing all found documents, in a concurrent way. Meaning, this single + /// process instance, will process multiple documents in parallel. + /// + /// Each document is loaded and processed within a dedicated transaction. Commiting the + /// transaction at the end each step and before moving on the next document. + /// + /// As handlers are intended to be idempotent, there's no harm in re-running them, in case + /// things go wrong. + /// + /// ## Caveats + /// + /// However, this may lead to a situation where only a part of the documents is processed. + /// But, this is ok, as the migration is supposed to run on a clone of the database and so the + /// actual system is still running from the read-only clone of the original data. async fn process( &self, storage: &DispatchBackend, @@ -188,6 +229,9 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { } /// A handler for data migration of documents. +/// +/// Handlers have to be written in a way that they can be re-run multiple times on the same +/// document without failing and creating the exact same output state. #[macro_export] macro_rules! handler { (async | $doc:ident: $doc_ty:ty, $model:ident, $tx:ident | $body:block) => {{ diff --git a/migration/src/data/partition.rs b/migration/src/data/partition.rs index 57d7fb9f0..37d599798 100644 --- a/migration/src/data/partition.rs +++ b/migration/src/data/partition.rs @@ -5,6 +5,7 @@ use std::{ }; use trustify_entity::{advisory, sbom}; +/// Information required for partitioning data #[derive(Debug, Copy, Clone)] pub struct Partition { pub current: u64, @@ -43,6 +44,9 @@ impl Default for Partition { } impl Partition { + /// Create a new partition of one. + /// + /// This will be one processor processing everything. pub const fn new_one() -> Self { Self { current: 0, From 4e1dea00a8521b1f8cb8c3625c91a49933d80d91 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Mon, 10 Nov 2025 12:07:44 +0100 Subject: [PATCH 42/47] refactor: make common function --- common/src/db/create.rs | 37 +++++++++++++++++++ common/src/db/mod.rs | 3 ++ migration/src/m0002010_add_advisory_scores.rs | 35 +----------------- 3 files changed, 41 insertions(+), 34 deletions(-) create mode 100644 common/src/db/create.rs diff --git a/common/src/db/create.rs b/common/src/db/create.rs new file mode 100644 index 000000000..bd72677c2 --- /dev/null +++ b/common/src/db/create.rs @@ -0,0 +1,37 @@ +use sea_orm::{ConnectionTrait, DbErr}; +use sea_orm_migration::SchemaManager; +use sea_query::{IntoIden, extension::postgres::Type}; + +/// create a type, if it not already exists +/// +/// This is required as Postgres doesn't support `CREATE TYPE IF NOT EXISTS` +pub async fn create_enum_if_not_exists( + manager: &SchemaManager<'_>, + name: impl IntoIden + Clone, + values: I, +) -> Result<(), DbErr> +where + T: IntoIden, + I: IntoIterator, +{ + let builder = manager.get_connection().get_database_backend(); + let r#type = name.clone().into_iden(); + let stmt = builder.build(Type::create().as_enum(name).values(values)); + let stmt = format!( + r#" +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type WHERE typname = '{name}' + ) THEN + {stmt}; + END IF; +END$$; +"#, + name = r#type.to_string() + ); + + manager.get_connection().execute_unprepared(&stmt).await?; + + Ok(()) +} diff --git a/common/src/db/mod.rs b/common/src/db/mod.rs index aadfdc8ad..9b0ba09a0 100644 --- a/common/src/db/mod.rs +++ b/common/src/db/mod.rs @@ -3,7 +3,10 @@ pub mod limiter; pub mod multi_model; pub mod query; +mod create; mod func; + +pub use create::*; pub use func::*; use anyhow::Context; diff --git a/migration/src/m0002010_add_advisory_scores.rs b/migration/src/m0002010_add_advisory_scores.rs index d76ef983b..ddd7629a1 100644 --- a/migration/src/m0002010_add_advisory_scores.rs +++ b/migration/src/m0002010_add_advisory_scores.rs @@ -5,6 +5,7 @@ use crate::{ use sea_orm::sea_query::extension::postgres::*; use sea_orm_migration::prelude::*; use strum::VariantNames; +use trustify_common::db::create_enum_if_not_exists; use trustify_module_ingestor::{ graph::cvss::ScoreCreator, service::advisory::{csaf, cve, osv}, @@ -13,40 +14,6 @@ use trustify_module_ingestor::{ #[derive(DeriveMigrationName)] pub struct Migration; -/// create a type, if it not already exists -/// -/// This is required as Postgres doesn't support `CREATE TYPE IF NOT EXISTS` -pub async fn create_enum_if_not_exists( - manager: &SchemaManager<'_>, - name: impl IntoIden + Clone, - values: I, -) -> Result<(), DbErr> -where - T: IntoIden, - I: IntoIterator, -{ - let builder = manager.get_connection().get_database_backend(); - let r#type = name.clone().into_iden(); - let stmt = builder.build(Type::create().as_enum(name).values(values)); - let stmt = format!( - r#" -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_type WHERE typname = '{name}' - ) THEN - {stmt}; - END IF; -END$$; -"#, - name = r#type.to_string() - ); - - manager.get_connection().execute_unprepared(&stmt).await?; - - Ok(()) -} - #[async_trait::async_trait] impl MigrationTraitWithData for Migration { async fn up(&self, manager: &SchemaDataManager) -> Result<(), DbErr> { From 4209b1dbcae1cf7e7df5636d009d3efdbe1d82c1 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Mon, 10 Nov 2025 14:54:43 +0100 Subject: [PATCH 43/47] refactor: simplify things a bit --- migration/src/data/mod.rs | 62 +++++-------------- migration/src/m0002000_add_sbom_properties.rs | 12 ++-- migration/src/m0002010_add_advisory_scores.rs | 18 +++--- migration/tests/data/main.rs | 8 +-- 4 files changed, 34 insertions(+), 66 deletions(-) diff --git a/migration/src/data/mod.rs b/migration/src/data/mod.rs index 8ee86eb0c..8e8ad784a 100644 --- a/migration/src/data/mod.rs +++ b/migration/src/data/mod.rs @@ -21,6 +21,7 @@ use std::{ }; use trustify_module_storage::service::dispatch::DispatchBackend; +/// A handler for processing a [`Document`] data migration. #[allow(async_fn_in_trait)] pub trait Handler: Send where @@ -34,6 +35,21 @@ where ) -> anyhow::Result<()>; } +impl Handler for F +where + D: Document, + for<'x> F: AsyncFn(D, D::Model, &'x DatabaseTransaction) -> anyhow::Result<()> + Send, +{ + async fn call( + &self, + document: D, + model: D::Model, + tx: &DatabaseTransaction, + ) -> anyhow::Result<()> { + (self)(document, model, tx).await + } +} + #[derive(Clone, Debug, PartialEq, Eq, clap::Parser)] pub struct Options { /// Number of concurrent documents being processes @@ -108,7 +124,7 @@ impl From<&Options> for Partition { } } -/// A trait for processing documents +/// A trait for processing documents using a [`Handler`]. pub trait DocumentProcessor { fn process( &self, @@ -228,50 +244,6 @@ impl<'c> DocumentProcessor for SchemaManager<'c> { } } -/// A handler for data migration of documents. -/// -/// Handlers have to be written in a way that they can be re-run multiple times on the same -/// document without failing and creating the exact same output state. -#[macro_export] -macro_rules! handler { - (async | $doc:ident: $doc_ty:ty, $model:ident, $tx:ident | $body:block) => {{ - struct H; - - impl $crate::data::Handler<$doc_ty> for H { - async fn call( - &self, - $doc: $doc_ty, - $model: <$doc_ty as $crate::data::Document>::Model, - $tx: &sea_orm::DatabaseTransaction, - ) -> anyhow::Result<()> { - $body - } - } - - H - }}; -} - -/// A handler for SBOMs. -/// -/// See: [`handler!`]. -#[macro_export] -macro_rules! sbom { - (async | $doc:ident, $model:ident, $tx:ident | $body:block) => { - $crate::handler!(async |$doc: $crate::data::Sbom, $model, $tx| $body) - }; -} - -/// A handler for advisories. -/// -/// See: [`handler!`]. -#[macro_export] -macro_rules! advisory { - (async | $doc:ident, $model:ident, $tx:ident | $body:block) => { - $crate::handler!(async |$doc: $crate::data::Advisory, $model, $tx| $body) - }; -} - pub trait MigratorWithData { fn data_migrations() -> Vec>; } diff --git a/migration/src/m0002000_add_sbom_properties.rs b/migration/src/m0002000_add_sbom_properties.rs index 603b24031..31dc7c696 100644 --- a/migration/src/m0002000_add_sbom_properties.rs +++ b/migration/src/m0002000_add_sbom_properties.rs @@ -1,10 +1,8 @@ -use crate::{ - data::{MigrationTraitWithData, Sbom as SbomDoc, SchemaDataManager}, - sbom, -}; -use sea_orm::{ActiveModelTrait, IntoActiveModel, Set}; +use crate::data::{MigrationTraitWithData, Sbom as SbomDoc, SchemaDataManager}; +use sea_orm::{ActiveModelTrait, DatabaseTransaction, IntoActiveModel, Set}; use sea_orm_migration::prelude::*; use trustify_common::advisory::cyclonedx::extract_properties_json; +use trustify_entity::sbom; #[derive(DeriveMigrationName)] pub struct Migration; @@ -38,7 +36,7 @@ impl MigrationTraitWithData for Migration { manager .process( self, - sbom!(async |sbom, model, tx| { + async |sbom: SbomDoc, model: sbom::Model, tx: &DatabaseTransaction| { let mut model = model.into_active_model(); match sbom { SbomDoc::CycloneDx(sbom) => { @@ -55,7 +53,7 @@ impl MigrationTraitWithData for Migration { model.save(tx).await?; Ok(()) - }), + }, ) .await?; diff --git a/migration/src/m0002010_add_advisory_scores.rs b/migration/src/m0002010_add_advisory_scores.rs index ddd7629a1..1b9e54f27 100644 --- a/migration/src/m0002010_add_advisory_scores.rs +++ b/migration/src/m0002010_add_advisory_scores.rs @@ -1,11 +1,9 @@ -use crate::{ - advisory, - data::{Advisory as AdvisoryDoc, MigrationTraitWithData, SchemaDataManager}, -}; -use sea_orm::sea_query::extension::postgres::*; +use crate::data::{Advisory, MigrationTraitWithData, SchemaDataManager}; +use sea_orm::{DatabaseTransaction, sea_query::extension::postgres::*}; use sea_orm_migration::prelude::*; use strum::VariantNames; use trustify_common::db::create_enum_if_not_exists; +use trustify_entity::advisory; use trustify_module_ingestor::{ graph::cvss::ScoreCreator, service::advisory::{csaf, cve, osv}, @@ -99,16 +97,16 @@ impl MigrationTraitWithData for Migration { manager .process( self, - advisory!(async |advisory, model, tx| { + async |advisory: Advisory, model: advisory::Model, tx: &DatabaseTransaction| { let mut creator = ScoreCreator::new(model.id); match advisory { - AdvisoryDoc::Cve(advisory) => { + Advisory::Cve(advisory) => { cve::extract_scores(&advisory, &mut creator); } - AdvisoryDoc::Csaf(advisory) => { + Advisory::Csaf(advisory) => { csaf::extract_scores(&advisory, &mut creator); } - AdvisoryDoc::Osv(advisory) => { + Advisory::Osv(advisory) => { osv::extract_scores(&advisory, &mut creator); } _ => { @@ -119,7 +117,7 @@ impl MigrationTraitWithData for Migration { creator.create(tx).await?; Ok(()) - }), + }, ) .await?; diff --git a/migration/tests/data/main.rs b/migration/tests/data/main.rs index c97f9871a..0e950e1bd 100644 --- a/migration/tests/data/main.rs +++ b/migration/tests/data/main.rs @@ -17,9 +17,9 @@ mod sbom { use migration::{ ColumnDef, DeriveIden, DeriveMigrationName, Table, async_trait, data::{MigrationTraitWithData, Sbom as SbomDoc, SchemaDataManager}, - sbom, }; - use sea_orm::{ConnectionTrait, DbErr, Statement}; + use sea_orm::{ConnectionTrait, DatabaseTransaction, DbErr, Statement}; + use trustify_entity::sbom; #[derive(DeriveMigrationName)] pub struct Migration; @@ -50,7 +50,7 @@ mod sbom { manager .process( self, - sbom!(async |sbom, model, tx| { + async |sbom: SbomDoc, model: sbom::Model, tx: &DatabaseTransaction| { // we just pick a random value let value = match sbom { SbomDoc::CycloneDx(sbom) => sbom.serial_number, @@ -70,7 +70,7 @@ mod sbom { } Ok(()) - }), + }, ) .await?; From 4c7ffc27991e37becb6d5cbea07566c0c88bc238 Mon Sep 17 00:00:00 2001 From: Dejan Bosanac Date: Tue, 23 Dec 2025 17:14:46 +0100 Subject: [PATCH 44/47] chore: upgrade to cvss-rs 0.2.0 --- Cargo.lock | 23 ++++++++++++----------- Cargo.toml | 2 +- entity/Cargo.toml | 2 +- modules/ingestor/Cargo.toml | 2 +- modules/ingestor/src/graph/cvss.rs | 17 ++++++----------- 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf42692b5..87af71013 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2200,20 +2200,21 @@ dependencies = [ [[package]] name = "cvss" -version = "0.1.0" -source = "git+https://github.com/dejanb/cvss#aadb35f033d124da099bf74fb1383fc30a62c163" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f643e062e9a8e26edea270945e05011c441ca6a56e9d9d4464c6b0be1352bd" + +[[package]] +name = "cvss-rs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac9b885fb8472719329455432c594b9344b6df6d9665aedb7f142193489bdcfd" dependencies = [ "serde", "serde_json", "strum 0.26.3", ] -[[package]] -name = "cvss" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f643e062e9a8e26edea270945e05011c441ca6a56e9d9d4464c6b0be1352bd" - [[package]] name = "darling" version = "0.10.2" @@ -8417,8 +8418,8 @@ dependencies = [ "anyhow", "async-graphql", "cpe", - "cvss 0.1.0", - "cvss 2.1.1", + "cvss", + "cvss-rs", "deepsize", "log", "rstest", @@ -8728,7 +8729,7 @@ dependencies = [ "cpe", "csaf", "cve", - "cvss 0.1.0", + "cvss-rs", "hex", "humantime", "jsn", diff --git a/Cargo.toml b/Cargo.toml index dbd98d13a..abc1b93c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ csaf = { version = "0.5.0", default-features = false } csaf-walker = { version = "0.14.1", default-features = false } csv = "1.3.0" cve = "0.5.0" -cvss-rs = "0.2" +cvss = { package = "cvss-rs", version = "0.2.0" } cvss-old = { package = "cvss", version = "2" } deepsize = "0.2.0" fixedbitset = "0.5.7" diff --git a/entity/Cargo.toml b/entity/Cargo.toml index c409e49f1..4bbbefbb4 100644 --- a/entity/Cargo.toml +++ b/entity/Cargo.toml @@ -13,7 +13,7 @@ trustify-common = { workspace = true } trustify-cvss = { workspace = true } cpe = { workspace = true } -cvss-rs = { workspace = true } +cvss = { workspace = true } cvss-old = { workspace = true } deepsize = { workspace = true } schemars = { workspace = true } diff --git a/modules/ingestor/Cargo.toml b/modules/ingestor/Cargo.toml index 1485c0e11..4fe24e7f3 100644 --- a/modules/ingestor/Cargo.toml +++ b/modules/ingestor/Cargo.toml @@ -19,7 +19,7 @@ bytes = { workspace = true } cpe = { workspace = true } csaf = { workspace = true } cve = { workspace = true } -cvss-rs = { workspace = true } +cvss = { workspace = true } hex = { workspace = true } humantime = { workspace = true } jsn = { workspace = true } diff --git a/modules/ingestor/src/graph/cvss.rs b/modules/ingestor/src/graph/cvss.rs index cdd537051..b16c84847 100644 --- a/modules/ingestor/src/graph/cvss.rs +++ b/modules/ingestor/src/graph/cvss.rs @@ -1,4 +1,4 @@ -use cvss::version::{VersionV2, VersionV3, VersionV4}; +use cvss::version::VersionV3; use cvss::{Cvss, v2_0, v3, v4_0}; use sea_orm::{ColumnTrait, ConnectionTrait, DbErr, EntityTrait, QueryFilter, Set}; use trustify_entity::advisory_vulnerability_score::{self, ScoreType, Severity}; @@ -44,7 +44,6 @@ impl From for advisory_vulnerability_score::ActiveModel { impl From<(String, v2_0::CvssV2)> for ScoreInformation { fn from((vulnerability_id, score): (String, v2_0::CvssV2)) -> Self { let v2_0::CvssV2 { - version, vector_string, severity, base_score, @@ -53,9 +52,7 @@ impl From<(String, v2_0::CvssV2)> for ScoreInformation { Self { vulnerability_id, - r#type: match version { - VersionV2::V2_0 => ScoreType::V2_0, - }, + r#type: ScoreType::V2_0, vector: vector_string, score: base_score, severity: match severity { @@ -81,8 +78,9 @@ impl From<(String, v3::CvssV3)> for ScoreInformation { Self { vulnerability_id, r#type: match version { - VersionV3::V3_0 => ScoreType::V3_0, - VersionV3::V3_1 => ScoreType::V3_1, + Some(VersionV3::V3_0) => ScoreType::V3_0, + Some(VersionV3::V3_1) => ScoreType::V3_1, + None => ScoreType::V3_0, // Default to V3_0 if version is not specified }, vector: vector_string, score: base_score, @@ -94,7 +92,6 @@ impl From<(String, v3::CvssV3)> for ScoreInformation { impl From<(String, v4_0::CvssV4)> for ScoreInformation { fn from((vulnerability_id, score): (String, v4_0::CvssV4)) -> Self { let v4_0::CvssV4 { - version, vector_string, base_severity, base_score, @@ -103,9 +100,7 @@ impl From<(String, v4_0::CvssV4)> for ScoreInformation { Self { vulnerability_id, - r#type: match version { - VersionV4::V4_0 => ScoreType::V4_0, - }, + r#type: ScoreType::V4_0, vector: vector_string, score: base_score, severity: base_severity.into(), From 2aff6e0ec09bd8d6aedddb0d4262d8426e7ff3b5 Mon Sep 17 00:00:00 2001 From: Dejan Bosanac Date: Thu, 8 Jan 2026 10:18:44 +0100 Subject: [PATCH 45/47] chore: upgrade csaf library Signed-off-by: Dejan Bosanac --- Cargo.lock | 15 ++++++++------- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87af71013..2a77732dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2090,7 +2090,7 @@ dependencies = [ [[package]] name = "csaf" version = "0.5.0" -source = "git+https://github.com/trustification/csaf-rs?branch=cvss#d6df319076cdc685a2cded0cb6797ad7ce92b9c9" +source = "git+https://github.com/trustification/csaf-rs?branch=main#4d65f7c791e16b7f80e0a3945bde9b91820f8f5b" dependencies = [ "chrono", "cpe", @@ -2188,9 +2188,9 @@ dependencies = [ [[package]] name = "cve" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bba109f3a468c9e2cd871c259040cf11ca95caaaed10427c6c8879515ae87896" +checksum = "3077b0b3df7108da3ff51db5c258785866d0c2392b7dd63c554b085d3bc075cc" dependencies = [ "serde", "serde_json", @@ -7796,9 +7796,9 @@ dependencies = [ [[package]] name = "test-context" -version = "0.4.1" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb69cce03e432993e2dc1f93f7899b952300fcb6dc44191a1b830b60b8c3c8aa" +checksum = "7d94db16dc1c321805ce55f286c4023fa58a2c9c742568f95c5cfe2e95d250d7" dependencies = [ "futures", "test-context-macros", @@ -7806,9 +7806,9 @@ dependencies = [ [[package]] name = "test-context-macros" -version = "0.4.1" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e0639209021e54dbe19cafabfc0b5574b078c37358945e6d473eabe39bb974" +checksum = "aabcca9d2cad192cfe258cd3562b7584516191a5c9b6a0002a6bb8b75ee7d21d" dependencies = [ "proc-macro2", "quote", @@ -8537,6 +8537,7 @@ dependencies = [ "packageurl", "parking_lot 0.12.4", "petgraph 0.8.2", + "rstest", "sea-orm", "sea-query", "serde", diff --git a/Cargo.toml b/Cargo.toml index abc1b93c1..2035c7596 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -207,7 +207,7 @@ postgresql_commands = { version = "0.20.0", default-features = false, features = # required due to https://github.com/KenDJohnson/cpe-rs/pull/15 #cpe = { git = "https://github.com/ctron/cpe-rs", rev = "c3c05e637f6eff7dd4933c2f56d070ee2ddfb44b" } # required due to https://github.com/voteblake/csaf-rs/pull/29 -csaf = { git = "https://github.com/trustification/csaf-rs", rev = "17620a225744b4a18845d4f7bf63354e01109b91" } +csaf = { git = "https://github.com/trustification/csaf-rs", branch = "main" } # required due to https://github.com/gcmurphy/osv/pull/58 #osv = { git = "https://github.com/ctron/osv", branch = "feature/drop_deps_1" } From d63185e903673f7c829cd16e6f3702dac7664cd6 Mon Sep 17 00:00:00 2001 From: Dejan Bosanac Date: Thu, 8 Jan 2026 10:35:02 +0100 Subject: [PATCH 46/47] chore: cleanup Signed-off-by: Dejan Bosanac --- migration/src/bin/data.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/migration/src/bin/data.rs b/migration/src/bin/data.rs index f67a6b8c6..b5f81966e 100644 --- a/migration/src/bin/data.rs +++ b/migration/src/bin/data.rs @@ -11,7 +11,6 @@ struct Cli { command: Command, } -#[allow(clippy::large_enum_variant)] #[derive(clap::Subcommand, Debug, Clone)] #[allow(clippy::large_enum_variant)] enum Command { From e88c7a57792067718f8bdc6dc48c5c50ce812199 Mon Sep 17 00:00:00 2001 From: Dejan Bosanac Date: Thu, 8 Jan 2026 13:56:09 +0100 Subject: [PATCH 47/47] feat: implement osv vector parsing Signed-off-by: Dejan Bosanac --- Cargo.lock | 2 +- entity/src/advisory_vulnerability_score.rs | 2 +- modules/ingestor/src/graph/cvss.rs | 81 ++++++----- .../src/service/advisory/osv/loader.rs | 31 ++++ .../ingestor/src/service/advisory/osv/mod.rs | 133 ++++++++---------- 5 files changed, 135 insertions(+), 114 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2a77732dd..182446ecb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -655,7 +655,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.106", ] [[package]] diff --git a/entity/src/advisory_vulnerability_score.rs b/entity/src/advisory_vulnerability_score.rs index 1cb1b53df..3c7a330c2 100644 --- a/entity/src/advisory_vulnerability_score.rs +++ b/entity/src/advisory_vulnerability_score.rs @@ -13,7 +13,7 @@ pub struct Model { pub r#type: ScoreType, pub vector: String, - pub score: f64, + pub score: f32, pub severity: Severity, } diff --git a/modules/ingestor/src/graph/cvss.rs b/modules/ingestor/src/graph/cvss.rs index b16c84847..d0b927192 100644 --- a/modules/ingestor/src/graph/cvss.rs +++ b/modules/ingestor/src/graph/cvss.rs @@ -1,6 +1,7 @@ use cvss::version::VersionV3; use cvss::{Cvss, v2_0, v3, v4_0}; use sea_orm::{ColumnTrait, ConnectionTrait, DbErr, EntityTrait, QueryFilter, Set}; +use trustify_cvss::cvss3::severity::Severity as CvssSeverity; use trustify_entity::advisory_vulnerability_score::{self, ScoreType, Severity}; use uuid::Uuid; @@ -16,7 +17,7 @@ pub struct ScoreInformation { pub vulnerability_id: String, pub r#type: ScoreType, pub vector: String, - pub score: f64, + pub score: f32, pub severity: Severity, } @@ -42,68 +43,72 @@ impl From for advisory_vulnerability_score::ActiveModel { } impl From<(String, v2_0::CvssV2)> for ScoreInformation { - fn from((vulnerability_id, score): (String, v2_0::CvssV2)) -> Self { - let v2_0::CvssV2 { - vector_string, - severity, - base_score, - .. - } = score; + fn from((vulnerability_id, cvss): (String, v2_0::CvssV2)) -> Self { + // Use calculated_base_score() to compute the actual score from metrics + let base_score = cvss.calculated_base_score().unwrap_or(0.0); + // Derive severity from calculated score using CVSS v2 scale (no "None" or "Critical") + let severity = match base_score { + x if x < 4.0 => Severity::Low, + x if x < 7.0 => Severity::Medium, + _ => Severity::High, + }; Self { vulnerability_id, r#type: ScoreType::V2_0, - vector: vector_string, - score: base_score, - severity: match severity { - None => Severity::None, - Some(v2_0::Severity::Low) => Severity::Low, - Some(v2_0::Severity::Medium) => Severity::Medium, - Some(v2_0::Severity::High) => Severity::High, - }, + vector: cvss.vector_string, + score: base_score as f32, + severity, } } } impl From<(String, v3::CvssV3)> for ScoreInformation { - fn from((vulnerability_id, score): (String, v3::CvssV3)) -> Self { - let v3::CvssV3 { - version, - vector_string, - base_severity, - base_score, - .. - } = score; + fn from((vulnerability_id, cvss): (String, v3::CvssV3)) -> Self { + // Use calculated_base_score() to compute the actual score from metrics + let base_score = cvss.calculated_base_score().unwrap_or(0.0); + // Derive severity from calculated score using CVSS v3 scale + let severity = match CvssSeverity::from_f64(base_score) { + CvssSeverity::None => Severity::None, + CvssSeverity::Low => Severity::Low, + CvssSeverity::Medium => Severity::Medium, + CvssSeverity::High => Severity::High, + CvssSeverity::Critical => Severity::Critical, + }; Self { vulnerability_id, - r#type: match version { + r#type: match cvss.version { Some(VersionV3::V3_0) => ScoreType::V3_0, Some(VersionV3::V3_1) => ScoreType::V3_1, None => ScoreType::V3_0, // Default to V3_0 if version is not specified }, - vector: vector_string, - score: base_score, - severity: base_severity.into(), + vector: cvss.vector_string, + score: base_score as f32, + severity, } } } impl From<(String, v4_0::CvssV4)> for ScoreInformation { - fn from((vulnerability_id, score): (String, v4_0::CvssV4)) -> Self { - let v4_0::CvssV4 { - vector_string, - base_severity, - base_score, - .. - } = score; + fn from((vulnerability_id, cvss): (String, v4_0::CvssV4)) -> Self { + // Use calculated_base_score() to compute the actual score from metrics + let base_score = cvss.calculated_base_score().unwrap_or(0.0); + // Derive severity from calculated score using CVSS v4 scale (same as v3) + let severity = match CvssSeverity::from_f64(base_score) { + CvssSeverity::None => Severity::None, + CvssSeverity::Low => Severity::Low, + CvssSeverity::Medium => Severity::Medium, + CvssSeverity::High => Severity::High, + CvssSeverity::Critical => Severity::Critical, + }; Self { vulnerability_id, r#type: ScoreType::V4_0, - vector: vector_string, - score: base_score, - severity: base_severity.into(), + vector: cvss.vector_string, + score: base_score as f32, + severity, } } } diff --git a/modules/ingestor/src/service/advisory/osv/loader.rs b/modules/ingestor/src/service/advisory/osv/loader.rs index 32409a317..b4bd8ea30 100644 --- a/modules/ingestor/src/service/advisory/osv/loader.rs +++ b/modules/ingestor/src/service/advisory/osv/loader.rs @@ -579,8 +579,10 @@ mod test { use hex::ToHex; use osv::schema::Vulnerability; use rstest::rstest; + use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; use test_context::test_context; use test_log::test; + use trustify_entity::advisory_vulnerability_score; use trustify_test_context::{TrustifyContext, document}; #[test_context(TrustifyContext)] @@ -642,6 +644,35 @@ mod test { .is_none() ); + // Verify the advisory_vulnerability_score table has the calculated score + let new_scores = advisory_vulnerability_score::Entity::find() + .filter( + advisory_vulnerability_score::Column::AdvisoryId.eq(loaded_advisory.advisory.id), + ) + .all(&ctx.db) + .await?; + assert_eq!(1, new_scores.len()); + let new_score = &new_scores[0]; + assert_eq!(new_score.vulnerability_id, "CVE-2021-32714"); + assert_eq!( + new_score.r#type, + advisory_vulnerability_score::ScoreType::V3_1 + ); + assert_eq!( + new_score.vector, + "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:H" + ); + // Score should be 9.1 (calculated from the CVSS vector metrics) + assert!( + (new_score.score - 9.1_f32).abs() < 0.1, + "Expected score ~9.1, got {}", + new_score.score + ); + assert_eq!( + new_score.severity, + advisory_vulnerability_score::Severity::Critical + ); + Ok(()) } diff --git a/modules/ingestor/src/service/advisory/osv/mod.rs b/modules/ingestor/src/service/advisory/osv/mod.rs index 51e9a602f..f65d29de9 100644 --- a/modules/ingestor/src/service/advisory/osv/mod.rs +++ b/modules/ingestor/src/service/advisory/osv/mod.rs @@ -3,12 +3,10 @@ mod prefix; pub mod loader; pub mod translate; -use crate::{ - graph::cvss::{ScoreCreator, ScoreInformation}, - service::Error, -}; +use crate::{graph::cvss::ScoreCreator, service::Error}; +use cvss::{v2_0::CvssV2, v3::CvssV3, v4_0::CvssV4}; use osv::schema::{SeverityType, Vulnerability}; -use trustify_entity::advisory_vulnerability_score::{ScoreType, Severity}; +use std::str::FromStr; /// Load a [`Vulnerability`] from YAML, using the "classic" enum representation. pub fn from_yaml(data: &[u8]) -> Result { @@ -49,78 +47,65 @@ pub fn extract_vulnerability_ids(osv: &Vulnerability) -> impl IntoIterator = extract_vulnerability_ids(osv).into_iter().collect(); + + // If no vulnerability IDs, nothing to do + if ids.is_empty() { + return; } - impl From<(String, ScoreInfo)> for ScoreInformation { - fn from( - ( - vulnerability_id, - ScoreInfo { - r#type, - vector, - score, - severity, - }, - ): (String, ScoreInfo), - ) -> Self { - Self { - vulnerability_id, - r#type, - vector, - score, - severity, + // Process each severity entry + for severity in osv.severity.iter().flatten() { + match severity.severity_type { + SeverityType::CVSSv2 => match CvssV2::from_str(&severity.score) { + Ok(cvss) => { + for id in &ids { + creator.add((id.to_string(), cvss.clone())); + } + } + Err(e) => { + log::warn!( + "Failed to parse CVSSv2 vector '{}': {:?}", + severity.score, + e + ); + } + }, + + SeverityType::CVSSv3 => match CvssV3::from_str(&severity.score) { + Ok(cvss) => { + for id in &ids { + creator.add((id.to_string(), cvss.clone())); + } + } + Err(e) => { + log::warn!( + "Failed to parse CVSSv3 vector '{}': {:?}", + severity.score, + e + ); + } + }, + + SeverityType::CVSSv4 => match CvssV4::from_str(&severity.score) { + Ok(cvss) => { + for id in &ids { + creator.add((id.to_string(), cvss.clone())); + } + } + Err(e) => { + log::warn!( + "Failed to parse CVSSv4 vector '{}': {:?}", + severity.score, + e + ); + } + }, + + _ => { + // Unknown severity type, skip } } } - - // TODO: validate score type by prefix - let scores = osv - .severity - .iter() - .flatten() - .flat_map(|severity| match severity.severity_type { - SeverityType::CVSSv2 => Some(ScoreInfo { - r#type: ScoreType::V2_0, - vector: severity.score.clone(), - score: 10f64, // TODO: replace with actual evaluated score - severity: Severity::Critical, // TODO: replace with actual evaluated severity - }), - SeverityType::CVSSv3 => Some(ScoreInfo { - r#type: match severity.score.starts_with("CVSS:3.1/") { - true => ScoreType::V3_1, - false => ScoreType::V3_0, - }, - vector: severity.score.clone(), - score: 10f64, // TODO: replace with actual evaluated score - severity: Severity::Critical, // TODO: replace with actual evaluated severity - }), - SeverityType::CVSSv4 => Some(ScoreInfo { - r#type: ScoreType::V4_0, - vector: severity.score.clone(), - score: 10f64, // TODO: replace with actual evaluated score - severity: Severity::Critical, // TODO: replace with actual evaluated severity - }), - - _ => None, - }); - - // get all vulnerability IDs - - let ids = extract_vulnerability_ids(osv) - .into_iter() - .collect::>(); - - // create scores for each vulnerability (alias) - - creator.extend( - scores - .into_iter() - .flat_map(|score| ids.iter().map(move |id| (id.to_string(), score.clone()))), - ); }