Skip to content

Commit

Permalink
Simplify the test
Browse files Browse the repository at this point in the history
Signed-off-by: Vlad Volodkin <vlaad@amazon.com>
  • Loading branch information
Vlad Volodkin committed Nov 13, 2024
1 parent b15094f commit 7a8c9b5
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 53 deletions.
4 changes: 2 additions & 2 deletions mountpoint-s3/tests/common/s3.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ pub fn get_test_bucket_and_prefix(test_name: &str) -> (String, String) {
#[cfg(not(feature = "s3express_tests"))]
let bucket = get_standard_bucket();
#[cfg(feature = "s3express_tests")]
let bucket = get_express_cache_bucket();
let bucket = get_express_bucket();

let prefix = get_test_prefix(test_name);

Expand All @@ -30,7 +30,7 @@ pub fn get_test_prefix(test_name: &str) -> String {
}

#[cfg(feature = "s3express_tests")]
pub fn get_express_cache_bucket() -> String {
pub fn get_express_bucket() -> String {
std::env::var("S3_EXPRESS_ONE_ZONE_BUCKET_NAME")
.expect("Set S3_EXPRESS_ONE_ZONE_BUCKET_NAME to run integration tests")
}
Expand Down
79 changes: 28 additions & 51 deletions mountpoint-s3/tests/fuse_tests/cache_test.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
use crate::common::fuse::s3_session::{create_crt_client, create_test_client};
use crate::common::fuse::{create_fuse_session, TestClient};
use crate::common::s3::{get_express_cache_bucket, get_standard_bucket, get_test_prefix, get_test_region};
use fuser::BackgroundSession;
use crate::common::s3::{get_express_bucket, get_standard_bucket, get_test_prefix, get_test_region};
use mountpoint_s3::data_cache::{DataCache, DiskDataCache, DiskDataCacheConfig, ExpressDataCache};
use mountpoint_s3::fs::CacheConfig;
use mountpoint_s3::prefetch::caching_prefetch;
Expand All @@ -10,10 +9,8 @@ use mountpoint_s3_client::S3CrtClient;
use rand::{Rng, RngCore, SeedableRng};
use rand_chacha::ChaChaRng;
use std::fs;
use std::path::PathBuf;
use std::thread::sleep;
use std::time::Duration;
use tempfile::TempDir;
use test_case::test_case;

const CACHE_BLOCK_SIZE: u64 = 1024 * 1024;
Expand All @@ -25,12 +22,16 @@ const CLIENT_PART_SIZE: usize = 8 * 1024 * 1024;
#[test_case("key", 100, 1024 * 1024; "big file")]
fn express_cache_write_read(key_suffix: &str, key_size: usize, object_size: usize) {
let client = create_crt_client(CLIENT_PART_SIZE, CLIENT_PART_SIZE);
let bucket_name = get_standard_bucket();
let express_bucket_name = get_express_bucket();
let cache = ExpressDataCache::new(&express_bucket_name, client.clone(), &bucket_name, CACHE_BLOCK_SIZE);

cache_write_read_base(
client.clone(),
client,
key_suffix,
key_size,
object_size,
express_cache_factory(client),
cache,
"express_cache_write_read",
)
}
Expand All @@ -41,27 +42,33 @@ fn express_cache_write_read(key_suffix: &str, key_size: usize, object_size: usiz
#[test_case("key", 100, 1024 * 1024; "big file")]
fn disk_cache_write_read(key_suffix: &str, key_size: usize, object_size: usize) {
let cache_dir = tempfile::tempdir().unwrap();
let cache_config = DiskDataCacheConfig {
block_size: CACHE_BLOCK_SIZE,
limit: Default::default(),
};
let cache = DiskDataCache::new(cache_dir.path().to_path_buf(), cache_config);

let client = create_crt_client(CLIENT_PART_SIZE, CLIENT_PART_SIZE);

cache_write_read_base(
client,
key_suffix,
key_size,
object_size,
disk_cache_factory(cache_dir.path().to_owned()),
cache,
"disk_cache_write_read",
);
}

fn cache_write_read_base<Cache, CacheFactory>(
fn cache_write_read_base<Cache>(
client: S3CrtClient,
key_suffix: &str,
key_size: usize,
object_size: usize,
cache_factory: CacheFactory,
cache: Cache,
test_name: &str,
) where
Cache: DataCache + Send + Sync + 'static,
CacheFactory: FnOnce() -> Cache,
{
let region = get_test_region();
let bucket = get_standard_bucket();
Expand All @@ -77,8 +84,17 @@ fn cache_write_read_base<Cache, CacheFactory>(
},
..Default::default()
};
let (mount_point, _session) =
new_fuse_session_with_cache(client, &bucket, &prefix, cache_factory, filesystem_config);
let mount_point = tempfile::tempdir().unwrap();
let runtime = client.event_loop_group();
let prefetcher = caching_prefetch(cache, runtime, Default::default());
let _session = create_fuse_session(
client,
prefetcher,
&bucket,
&prefix,
mount_point.path(),
filesystem_config,
);

// Write an object, no caching happens yet
let key = get_object_key(&prefix, key_suffix, key_size);
Expand Down Expand Up @@ -111,23 +127,6 @@ fn cache_write_read_base<Cache, CacheFactory>(
assert_eq!(read, written);
}

fn express_cache_factory(client: S3CrtClient) -> impl FnOnce() -> ExpressDataCache<S3CrtClient> {
move || {
let express_bucket_name = get_express_cache_bucket();
ExpressDataCache::new(&express_bucket_name, client, &express_bucket_name, CACHE_BLOCK_SIZE)
}
}

fn disk_cache_factory(cache_dir: PathBuf) -> impl FnOnce() -> DiskDataCache {
move || {
let cache_config = DiskDataCacheConfig {
block_size: CACHE_BLOCK_SIZE,
limit: Default::default(),
};
DiskDataCache::new(cache_dir, cache_config)
}
}

fn random_binary_data(size_in_bytes: usize) -> Vec<u8> {
let seed = rand::thread_rng().gen();
let mut rng = ChaChaRng::seed_from_u64(seed);
Expand All @@ -146,25 +145,3 @@ fn get_object_key(key_prefix: &str, key_suffix: &str, min_size_in_bytes: usize)
let padding = "0".repeat(padding_size);
format!("{last_key_part}{padding}")
}

/// Create a FUSE mount backed by a real S3 client with a cache.
/// Note, that the mount uses S3 Standard as a source bucket.
fn new_fuse_session_with_cache<Cache, CacheFactory>(
client: S3CrtClient,
bucket: &str,
prefix: &str,
cache_factory: CacheFactory,
filesystem_config: S3FilesystemConfig,
) -> (TempDir, BackgroundSession)
where
Cache: DataCache + Send + Sync + 'static,
CacheFactory: FnOnce() -> Cache,
{
let mount_dir = tempfile::tempdir().unwrap();
let cache = cache_factory();
let runtime = client.event_loop_group();
let prefetcher = caching_prefetch(cache, runtime, Default::default());
let session = create_fuse_session(client, prefetcher, bucket, prefix, mount_dir.path(), filesystem_config);

(mount_dir, session)
}

0 comments on commit 7a8c9b5

Please sign in to comment.