From b03aa9929f415997984d4d0d385f624df1dfec56 Mon Sep 17 00:00:00 2001 From: shimun Date: Mon, 17 Jul 2023 15:00:23 +0200 Subject: [PATCH] feat(push): filter derivation by regex --- attic/src/nix_store/nix_store.rs | 17 ++++++++++++----- attic/src/nix_store/tests/mod.rs | 2 +- client/src/command/push.rs | 12 +++++++++++- client/src/push.rs | 18 ++++++++++++++---- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/attic/src/nix_store/nix_store.rs b/attic/src/nix_store/nix_store.rs index 79a3fc7..bbab166 100644 --- a/attic/src/nix_store/nix_store.rs +++ b/attic/src/nix_store/nix_store.rs @@ -5,12 +5,13 @@ use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; use std::sync::Arc; +use regex::Regex; use tokio::task::spawn_blocking; use super::bindings::{open_nix_store, AsyncWriteAdapter, FfiNixStore}; use super::{to_base_name, StorePath, ValidPathInfo}; -use crate::hash::Hash; use crate::error::AtticResult; +use crate::hash::Hash; /// High-level wrapper for the Unix Domain Socket Nix Store. pub struct NixStore { @@ -156,6 +157,7 @@ impl NixStore { flip_directions: bool, include_outputs: bool, include_derivers: bool, + filter: Option, ) -> AtticResult> { let inner = self.inner.clone(); @@ -174,16 +176,20 @@ impl NixStore { Ok(cxx_vector .iter() - .map(|s| { + .filter_map(|s| { let osstr = OsStr::from_bytes(s.as_bytes()); let pb = PathBuf::from(osstr); // Safety: The C++ implementation already checks the StorePath // for correct format (which also implies valid UTF-8) #[allow(unsafe_code)] - unsafe { - StorePath::from_base_name_unchecked(pb) + let store_path = unsafe { StorePath::from_base_name_unchecked(pb) }; + if let Some(ref filter) = filter { + if filter.is_match(&store_path.name()) { + return None; + } } + Some(store_path) }) .collect()) }) @@ -201,7 +207,8 @@ impl NixStore { // FIXME: Make this more ergonomic and efficient let nar_size = c_path_info.pin_mut().nar_size(); - let nar_sha256_hash: [u8; 32] = c_path_info.pin_mut().nar_sha256_hash().try_into().unwrap(); + let nar_sha256_hash: [u8; 32] = + c_path_info.pin_mut().nar_sha256_hash().try_into().unwrap(); let references = c_path_info .pin_mut() .references() diff --git a/attic/src/nix_store/tests/mod.rs b/attic/src/nix_store/tests/mod.rs index 3d23922..d57c32b 100644 --- a/attic/src/nix_store/tests/mod.rs +++ b/attic/src/nix_store/tests/mod.rs @@ -212,7 +212,7 @@ fn test_compute_fs_closure_multi() { ]; let actual: HashSet = store - .compute_fs_closure_multi(paths, false, false, false) + .compute_fs_closure_multi(paths, false, false, false, None) .await .expect("Could not compute closure") .into_iter() diff --git a/client/src/command/push.rs b/client/src/command/push.rs index 9e1e795..5f8aa70 100644 --- a/client/src/command/push.rs +++ b/client/src/command/push.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; use clap::Parser; use indicatif::MultiProgress; +use regex::Regex; use crate::api::ApiClient; use crate::cache::CacheRef; @@ -24,6 +25,10 @@ pub struct Push { /// The store paths to push. paths: Vec, + /// Derivation names to be filted out. + #[clap(long)] + filter: Option, + /// Push the specified paths only and do not compute closures. #[clap(long)] no_closure: bool, @@ -78,7 +83,12 @@ pub async fn run(opts: Opts) -> Result<()> { let pusher = Pusher::new(store, api, cache.to_owned(), cache_config, mp, push_config); let plan = pusher - .plan(roots, sub.no_closure, sub.ignore_upstream_cache_filter) + .plan( + roots, + sub.no_closure, + sub.ignore_upstream_cache_filter, + sub.filter.clone(), + ) .await?; if plan.store_path_map.is_empty() { diff --git a/client/src/push.rs b/client/src/push.rs index 1b7194f..1916465 100644 --- a/client/src/push.rs +++ b/client/src/push.rs @@ -28,6 +28,7 @@ use bytes::Bytes; use futures::future::join_all; use futures::stream::{Stream, TryStreamExt}; use indicatif::{HumanBytes, MultiProgress, ProgressBar, ProgressState, ProgressStyle}; +use regex::Regex; use tokio::sync::Mutex; use tokio::task::{spawn, JoinHandle}; use tokio::time; @@ -192,6 +193,7 @@ impl Pusher { roots: Vec, no_closure: bool, ignore_upstream_filter: bool, + filter: Option, ) -> Result { PushPlan::plan( self.store.clone(), @@ -201,6 +203,7 @@ impl Pusher { roots, no_closure, ignore_upstream_filter, + filter, ) .await } @@ -336,6 +339,7 @@ impl PushSession { roots_vec, config.no_closure, config.ignore_upstream_cache_filter, + None, ) .await?; @@ -375,28 +379,34 @@ impl PushPlan { roots: Vec, no_closure: bool, ignore_upstream_filter: bool, + filter: Option, ) -> Result { // Compute closure let closure = if no_closure { roots } else { store - .compute_fs_closure_multi(roots, false, false, false) + .compute_fs_closure_multi(roots, false, false, false, None) .await? }; let mut store_path_map: HashMap = { let futures = closure .iter() - .map(|path| { + .flat_map(|path| { let store = store.clone(); let path = path.clone(); + if let Some(ref filter) = filter { + if filter.is_match(&path.name()) { + return None; + } + } let path_hash = path.to_hash(); - async move { + Some(async move { let path_info = store.query_path_info(path).await?; Ok((path_hash, path_info)) - } + }) }) .collect::>();