diff --git a/mountpoint-s3-client/CHANGELOG.md b/mountpoint-s3-client/CHANGELOG.md index 4bf56ff2a..238e8701b 100644 --- a/mountpoint-s3-client/CHANGELOG.md +++ b/mountpoint-s3-client/CHANGELOG.md @@ -2,7 +2,9 @@ ### Other changes -* No other changes. +* Add parameter to request checksum information as part of a `HeadObject` request. + If specified, the result should contain the checksum for the object if available in the S3 response. + ([#1083](https://github.com/awslabs/mountpoint-s3/pull/1083)) ### Breaking changes @@ -12,6 +14,9 @@ ([#1058](https://github.com/awslabs/mountpoint-s3/pull/1058)) * `HeadObjectResult` no longer provides the bucket and key used in the original request. ([#1058](https://github.com/awslabs/mountpoint-s3/pull/1058)) +* `head_object` method now requires a `HeadObjectParams` parameter. + The structure itself is not required to specify anything to achieve the existing behavior. + ([#1083](https://github.com/awslabs/mountpoint-s3/pull/1083)) ## v0.11.0 (October 17, 2024) diff --git a/mountpoint-s3-client/src/failure_client.rs b/mountpoint-s3-client/src/failure_client.rs index 24d8c8108..62dc20069 100644 --- a/mountpoint-s3-client/src/failure_client.rs +++ b/mountpoint-s3-client/src/failure_client.rs @@ -17,9 +17,9 @@ use pin_project::pin_project; use crate::object_client::{ CopyObjectError, CopyObjectParams, CopyObjectResult, DeleteObjectError, DeleteObjectResult, ETag, GetBodyPart, GetObjectAttributesError, GetObjectAttributesResult, GetObjectError, GetObjectRequest, HeadObjectError, - HeadObjectResult, ListObjectsError, ListObjectsResult, ObjectAttribute, ObjectClient, ObjectClientError, - ObjectClientResult, PutObjectError, PutObjectParams, PutObjectRequest, PutObjectResult, PutObjectSingleParams, - UploadReview, + HeadObjectParams, HeadObjectResult, ListObjectsError, ListObjectsResult, ObjectAttribute, ObjectClient, + ObjectClientError, ObjectClientResult, PutObjectError, PutObjectParams, PutObjectRequest, PutObjectResult, + PutObjectSingleParams, UploadReview, }; // Wrapper for injecting failures into a get stream or a put request @@ -167,9 +167,10 @@ where &self, bucket: &str, key: &str, + params: &HeadObjectParams, ) -> ObjectClientResult { (self.head_object_cb)(&mut *self.state.lock().unwrap(), bucket, key)?; - self.client.head_object(bucket, key).await + self.client.head_object(bucket, key, params).await } async fn put_object( diff --git a/mountpoint-s3-client/src/lib.rs b/mountpoint-s3-client/src/lib.rs index 861ff9739..2aa3c318e 100644 --- a/mountpoint-s3-client/src/lib.rs +++ b/mountpoint-s3-client/src/lib.rs @@ -72,11 +72,11 @@ pub mod config { /// Types used by all object clients pub mod types { pub use super::object_client::{ - Checksum, ChecksumAlgorithm, CopyObjectParams, CopyObjectResult, DeleteObjectResult, ETag, GetBodyPart, - GetObjectAttributesParts, GetObjectAttributesResult, GetObjectRequest, HeadObjectResult, ListObjectsResult, - ObjectAttribute, ObjectClientResult, ObjectInfo, ObjectPart, PutObjectParams, PutObjectResult, - PutObjectSingleParams, PutObjectTrailingChecksums, RestoreStatus, UploadChecksum, UploadReview, - UploadReviewPart, + Checksum, ChecksumAlgorithm, ChecksumMode, CopyObjectParams, CopyObjectResult, DeleteObjectResult, ETag, + GetBodyPart, GetObjectAttributesParts, GetObjectAttributesResult, GetObjectRequest, HeadObjectParams, + HeadObjectResult, ListObjectsResult, ObjectAttribute, ObjectClientResult, ObjectInfo, ObjectPart, + PutObjectParams, PutObjectResult, PutObjectSingleParams, PutObjectTrailingChecksums, RestoreStatus, + UploadChecksum, UploadReview, UploadReviewPart, }; } diff --git a/mountpoint-s3-client/src/mock_client.rs b/mountpoint-s3-client/src/mock_client.rs index d2bd442c3..06b37b049 100644 --- a/mountpoint-s3-client/src/mock_client.rs +++ b/mountpoint-s3-client/src/mock_client.rs @@ -25,11 +25,11 @@ use tracing::trace; use crate::checksums::crc32c_to_base64; use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata}; use crate::object_client::{ - Checksum, ChecksumAlgorithm, CopyObjectError, CopyObjectParams, CopyObjectResult, DeleteObjectError, + Checksum, ChecksumAlgorithm, ChecksumMode, CopyObjectError, CopyObjectParams, CopyObjectResult, DeleteObjectError, DeleteObjectResult, ETag, GetBodyPart, GetObjectAttributesError, GetObjectAttributesParts, - GetObjectAttributesResult, GetObjectError, GetObjectRequest, HeadObjectError, HeadObjectResult, ListObjectsError, - ListObjectsResult, ObjectAttribute, ObjectClient, ObjectClientError, ObjectClientResult, ObjectInfo, ObjectPart, - PutObjectError, PutObjectParams, PutObjectRequest, PutObjectResult, PutObjectSingleParams, + GetObjectAttributesResult, GetObjectError, GetObjectRequest, HeadObjectError, HeadObjectParams, HeadObjectResult, + ListObjectsError, ListObjectsResult, ObjectAttribute, ObjectClient, ObjectClientError, ObjectClientResult, + ObjectInfo, ObjectPart, PutObjectError, PutObjectParams, PutObjectRequest, PutObjectResult, PutObjectSingleParams, PutObjectTrailingChecksums, RestoreStatus, UploadReview, UploadReviewPart, }; @@ -376,6 +376,10 @@ pub struct MockObject { etag: ETag, parts: Option, object_metadata: HashMap, + /// S3 checksums associated with the object. + /// + /// Typically, at most one of the checksums should be set. + checksum: Checksum, } impl MockObject { @@ -395,6 +399,7 @@ impl MockObject { etag, parts: None, object_metadata: HashMap::new(), + checksum: Checksum::empty(), } } @@ -408,6 +413,7 @@ impl MockObject { etag, parts: None, object_metadata: HashMap::new(), + checksum: Checksum::empty(), } } @@ -431,6 +437,7 @@ impl MockObject { etag, parts: None, object_metadata: HashMap::new(), + checksum: Checksum::empty(), } } @@ -450,6 +457,10 @@ impl MockObject { self.restore_status = restore_status; } + pub fn set_checksum(&mut self, checksum: Checksum) { + self.checksum = checksum; + } + pub fn len(&self) -> usize { self.size } @@ -676,6 +687,7 @@ impl ObjectClient for MockClient { &self, bucket: &str, key: &str, + params: &HeadObjectParams, ) -> ObjectClientResult { trace!(bucket, key, "HeadObject"); self.inc_op_count(Operation::HeadObject); @@ -686,12 +698,19 @@ impl ObjectClient for MockClient { let objects = self.objects.read().unwrap(); if let Some(object) = objects.get(key) { + // Checksum information is opt-in + let checksum = match params.checksum_mode { + Some(ChecksumMode::Enabled) => object.checksum.clone(), + None => Checksum::empty(), + }; + Ok(HeadObjectResult { size: object.size as u64, last_modified: object.last_modified, etag: object.etag.clone(), storage_class: object.storage_class.clone(), restore_status: object.restore_status, + checksum, }) } else { Err(ObjectClientError::ServiceError(HeadObjectError::NotFound)) @@ -1676,7 +1695,7 @@ mod tests { put_request.complete().await.unwrap(); // head_object returns storage class - let head_result = client.head_object(bucket, key).await.unwrap(); + let head_result = client.head_object(bucket, key, &HeadObjectParams::new()).await.unwrap(); assert_eq!(head_result.storage_class.as_deref(), storage_class); // list_objects returns storage class @@ -1699,14 +1718,14 @@ mod tests { let head_counter_1 = client.new_counter(Operation::HeadObject); let delete_counter_1 = client.new_counter(Operation::DeleteObject); - let _result = client.head_object(bucket, "key").await; + let _result = client.head_object(bucket, "key", &HeadObjectParams::new()).await; assert_eq!(1, head_counter_1.count()); assert_eq!(0, delete_counter_1.count()); let head_counter_2 = client.new_counter(Operation::HeadObject); assert_eq!(0, head_counter_2.count()); - let _result = client.head_object(bucket, "key").await; + let _result = client.head_object(bucket, "key", &HeadObjectParams::new()).await; let _result = client.delete_object(bucket, "key").await; let _result = client.delete_object(bucket, "key").await; let _result = client.delete_object(bucket, "key").await; diff --git a/mountpoint-s3-client/src/mock_client/throughput_client.rs b/mountpoint-s3-client/src/mock_client/throughput_client.rs index bf651a67c..39958e768 100644 --- a/mountpoint-s3-client/src/mock_client/throughput_client.rs +++ b/mountpoint-s3-client/src/mock_client/throughput_client.rs @@ -16,8 +16,8 @@ use crate::mock_client::{ use crate::object_client::{ CopyObjectError, CopyObjectParams, CopyObjectResult, DeleteObjectError, DeleteObjectResult, ETag, GetBodyPart, GetObjectAttributesError, GetObjectAttributesResult, GetObjectError, GetObjectRequest, HeadObjectError, - HeadObjectResult, ListObjectsError, ListObjectsResult, ObjectAttribute, ObjectClient, ObjectClientResult, - PutObjectError, PutObjectParams, PutObjectResult, PutObjectSingleParams, + HeadObjectParams, HeadObjectResult, ListObjectsError, ListObjectsResult, ObjectAttribute, ObjectClient, + ObjectClientResult, PutObjectError, PutObjectParams, PutObjectResult, PutObjectSingleParams, }; /// A [MockClient] that rate limits overall download throughput to simulate a target network @@ -168,8 +168,9 @@ impl ObjectClient for ThroughputMockClient { &self, bucket: &str, key: &str, + params: &HeadObjectParams, ) -> ObjectClientResult { - self.inner.head_object(bucket, key).await + self.inner.head_object(bucket, key, params).await } async fn put_object( diff --git a/mountpoint-s3-client/src/object_client.rs b/mountpoint-s3-client/src/object_client.rs index b5db33387..a6ce3bab2 100644 --- a/mountpoint-s3-client/src/object_client.rs +++ b/mountpoint-s3-client/src/object_client.rs @@ -97,6 +97,7 @@ pub trait ObjectClient { &self, bucket: &str, key: &str, + params: &HeadObjectParams, ) -> ObjectClientResult; /// Put an object into the object store. Returns a [PutObjectRequest] for callers @@ -206,6 +207,35 @@ pub enum ListObjectsError { NoSuchBucket, } +/// Parameters to a [`head_object`](ObjectClient::head_object) request +#[derive(Debug, Default, Clone)] +#[non_exhaustive] +pub struct HeadObjectParams { + /// Enable to retrieve checksum as part of the HeadObject request + pub checksum_mode: Option, +} + +impl HeadObjectParams { + /// Create a default [HeadObjectParams]. + pub fn new() -> Self { + Self::default() + } + + /// Set option to retrieve checksum as part of the HeadObject request + pub fn checksum_mode(mut self, value: Option) -> Self { + self.checksum_mode = value; + self + } +} + +/// Enable [ChecksumMode] to retrieve object checksums +#[non_exhaustive] +#[derive(Clone, Debug)] +pub enum ChecksumMode { + /// Retrieve checksums + Enabled, +} + /// Result of a [`head_object`](ObjectClient::head_object) request #[derive(Debug)] #[non_exhaustive] @@ -232,6 +262,11 @@ pub struct HeadObjectResult { /// Objects in flexible retrieval storage classes (such as GLACIER and DEEP_ARCHIVE) are only /// accessible after restoration pub restore_status: Option, + /// Checksum of the object. + /// + /// HeadObject must explicitly request for this field to be included, + /// otherwise the values will be empty. + pub checksum: Checksum, } /// Errors returned by a [`head_object`](ObjectClient::head_object) request @@ -647,7 +682,7 @@ impl fmt::Display for ObjectAttribute { /// /// See [Checksum](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Checksum.html) in the *Amazon /// S3 API Reference* for more details. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct Checksum { /// Base64-encoded, 32-bit CRC32 checksum of the object pub checksum_crc32: Option, @@ -662,6 +697,18 @@ pub struct Checksum { pub checksum_sha256: Option, } +impl Checksum { + /// Construct an empty [Checksum] + pub fn empty() -> Self { + Self { + checksum_crc32: None, + checksum_crc32c: None, + checksum_sha1: None, + checksum_sha256: None, + } + } +} + /// Metadata about object parts from GetObjectAttributes API. /// /// See [GetObjectAttributesParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAttributesParts.html) diff --git a/mountpoint-s3-client/src/s3_crt_client.rs b/mountpoint-s3-client/src/s3_crt_client.rs index 6e1b811f8..9c25a091c 100644 --- a/mountpoint-s3-client/src/s3_crt_client.rs +++ b/mountpoint-s3-client/src/s3_crt_client.rs @@ -1305,8 +1305,9 @@ impl ObjectClient for S3CrtClient { &self, bucket: &str, key: &str, + params: &HeadObjectParams, ) -> ObjectClientResult { - self.head_object(bucket, key).await + self.head_object(bucket, key, params).await } async fn put_object( diff --git a/mountpoint-s3-client/src/s3_crt_client/head_object.rs b/mountpoint-s3-client/src/s3_crt_client/head_object.rs index 9beba4f54..d979c22e7 100644 --- a/mountpoint-s3-client/src/s3_crt_client/head_object.rs +++ b/mountpoint-s3-client/src/s3_crt_client/head_object.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use std::sync::{Arc, Mutex}; use lazy_static::lazy_static; -use mountpoint_s3_crt::http::request_response::{Headers, HeadersError}; +use mountpoint_s3_crt::http::request_response::{Header, Headers, HeadersError}; use mountpoint_s3_crt::s3::client::MetaRequestResult; use regex::Regex; use thiserror::Error; @@ -11,9 +11,13 @@ use time::format_description::well_known::Rfc2822; use time::OffsetDateTime; use tracing::error; -use crate::object_client::{HeadObjectError, HeadObjectResult, ObjectClientError, ObjectClientResult, RestoreStatus}; +use crate::object_client::{ + Checksum, HeadObjectError, HeadObjectParams, HeadObjectResult, ObjectClientError, ObjectClientResult, RestoreStatus, +}; use crate::s3_crt_client::{S3CrtClient, S3Operation, S3RequestError}; +use super::ChecksumMode; + #[derive(Error, Debug)] #[non_exhaustive] pub enum ParseError { @@ -83,6 +87,20 @@ impl HeadObjectResult { Ok(Some(RestoreStatus::Restored { expiry: expiry.into() })) } + fn parse_checksum(headers: &Headers) -> Result { + let checksum_crc32 = get_optional_field(headers, "x-amz-checksum-crc32")?; + let checksum_crc32c = get_optional_field(headers, "x-amz-checksum-crc32c")?; + let checksum_sha1 = get_optional_field(headers, "x-amz-checksum-sha1")?; + let checksum_sha256 = get_optional_field(headers, "x-amz-checksum-sha256")?; + + Ok(Checksum { + checksum_crc32, + checksum_crc32c, + checksum_sha1, + checksum_sha256, + }) + } + /// Parse from HeadObject headers fn parse_from_hdr(headers: &Headers) -> Result { let last_modified = OffsetDateTime::parse(&get_field(headers, "Last-Modified")?, &Rfc2822) @@ -92,12 +110,14 @@ impl HeadObjectResult { let etag = get_field(headers, "Etag")?; let storage_class = get_optional_field(headers, "x-amz-storage-class")?; let restore_status = Self::parse_restore_status(headers)?; + let checksum = Self::parse_checksum(headers)?; let result = HeadObjectResult { size, last_modified, storage_class, restore_status, etag: etag.into(), + checksum, }; Ok(result) } @@ -108,6 +128,7 @@ impl S3CrtClient { &self, bucket: &str, key: &str, + params: &HeadObjectParams, ) -> ObjectClientResult { // Stash the response from the head_object in this lock during the on_headers // callback, and pull them out once the request is done. @@ -127,6 +148,17 @@ impl S3CrtClient { let bucket = bucket.to_owned(); + match params.checksum_mode { + Some(ChecksumMode::Enabled) => { + message + .set_header(&Header::new("x-amz-checksum-mode", "ENABLED")) + .map_err(S3RequestError::construction_failure)?; + } + None => { + // No-op. Leaving this branch so new variants will cause compilation to fail. + } + } + let span = request_span!(self.inner, "head_object", bucket, key); self.inner.make_meta_request( @@ -224,6 +256,24 @@ mod tests { }; } + #[test] + fn test_checksum_sha256() { + let mut headers = Headers::new(&Allocator::default()).unwrap(); + let value = "QwzjTQIHJO11oZbfwq1nx3dy0Wk="; + let header = Header::new("x-amz-checksum-sha256", value.to_owned()); + headers.add_header(&header).unwrap(); + + let checksum = HeadObjectResult::parse_checksum(&headers).expect("failed to parse headers"); + assert_eq!(checksum.checksum_crc32, None, "other checksums shouldn't be set"); + assert_eq!(checksum.checksum_crc32c, None, "other checksums shouldn't be set"); + assert_eq!(checksum.checksum_sha1, None, "other checksums shouldn't be set"); + assert_eq!( + checksum.checksum_sha256, + Some(value.to_owned()), + "sha256 header should match" + ); + } + #[test] fn test_parse_restore_empty() { let headers = Headers::new(&Allocator::default()).unwrap(); diff --git a/mountpoint-s3-client/tests/head_object.rs b/mountpoint-s3-client/tests/head_object.rs index 7e54a59b6..5b48a0dba 100644 --- a/mountpoint-s3-client/tests/head_object.rs +++ b/mountpoint-s3-client/tests/head_object.rs @@ -6,6 +6,7 @@ pub mod common; use std::time::{Duration, Instant}; use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::ChecksumAlgorithm; #[cfg(not(feature = "s3express_tests"))] use aws_sdk_s3::types::{GlacierJobParameters, RestoreRequest, Tier}; use bytes::Bytes; @@ -13,8 +14,8 @@ use common::*; use mountpoint_s3_client::error::{HeadObjectError, ObjectClientError}; #[cfg(not(feature = "s3express_tests"))] use mountpoint_s3_client::types::RestoreStatus; +use mountpoint_s3_client::types::{ChecksumMode, HeadObjectParams}; use mountpoint_s3_client::{ObjectClient, S3CrtClient, S3RequestError}; -#[cfg(not(feature = "s3express_tests"))] use test_case::test_case; #[tokio::test] @@ -34,7 +35,10 @@ async fn test_head_object() { .unwrap(); let client: S3CrtClient = get_test_client(); - let result = client.head_object(&bucket, &key).await.expect("head_object failed"); + let result = client + .head_object(&bucket, &key, &HeadObjectParams::new()) + .await + .expect("head_object failed"); assert_eq!( result.size as usize, @@ -43,6 +47,69 @@ async fn test_head_object() { ); } +#[test_case(ChecksumAlgorithm::Crc32)] +#[test_case(ChecksumAlgorithm::Crc32C)] +#[test_case(ChecksumAlgorithm::Sha1)] +#[test_case(ChecksumAlgorithm::Sha256)] +#[tokio::test] +async fn test_head_object_checksum(checksum_algorithm: ChecksumAlgorithm) { + let sdk_client = get_test_sdk_client().await; + let (bucket, prefix) = get_test_bucket_and_prefix("test_head_object"); + + let key = format!("{prefix}hello"); + let body = b"hello world!"; + let put_object_output = sdk_client + .put_object() + .bucket(&bucket) + .key(&key) + .body(ByteStream::from(Bytes::from_static(body))) + .checksum_algorithm(checksum_algorithm.clone()) + .send() + .await + .unwrap(); + + let client: S3CrtClient = get_test_client(); + + for retrieve_checksum in [true, false] { + let mut params = HeadObjectParams::new(); + if retrieve_checksum { + params = params.checksum_mode(Some(ChecksumMode::Enabled)); + } + let result = client + .head_object(&bucket, &key, ¶ms) + .await + .expect("head_object failed"); + + let checksum = result.checksum; + if retrieve_checksum { + match &checksum_algorithm { + ChecksumAlgorithm::Crc32 => assert_eq!( + checksum.checksum_crc32, + put_object_output.checksum_crc32().map(|s| s.to_string()) + ), + ChecksumAlgorithm::Crc32C => assert_eq!( + checksum.checksum_crc32c, + put_object_output.checksum_crc32_c().map(|s| s.to_string()) + ), + ChecksumAlgorithm::Sha1 => assert_eq!( + checksum.checksum_sha1, + put_object_output.checksum_sha1().map(|s| s.to_string()) + ), + ChecksumAlgorithm::Sha256 => assert_eq!( + checksum.checksum_sha256, + put_object_output.checksum_sha256().map(|s| s.to_string()) + ), + _ => unimplemented!("This algorithm is not supported"), + } + } else { + assert!(checksum.checksum_crc32.is_none()); + assert!(checksum.checksum_crc32c.is_none()); + assert!(checksum.checksum_sha1.is_none()); + assert!(checksum.checksum_sha256.is_none()); + } + } +} + #[test_case("INTELLIGENT_TIERING")] #[test_case("GLACIER")] #[tokio::test] @@ -52,7 +119,7 @@ async fn test_head_object_storage_class(storage_class: &str) { let sdk_client = get_test_sdk_client().await; let (bucket, prefix) = get_test_bucket_and_prefix("test_head_object"); - let key = format!("{prefix}/hello"); + let key = format!("{prefix}hello"); let body = b"hello world!"; sdk_client .put_object() @@ -65,7 +132,10 @@ async fn test_head_object_storage_class(storage_class: &str) { .unwrap(); let client: S3CrtClient = get_test_client(); - let result = client.head_object(&bucket, &key).await.expect("head_object failed"); + let result = client + .head_object(&bucket, &key, &HeadObjectParams::new()) + .await + .expect("head_object failed"); assert_eq!(result.size as usize, body.len()); assert_eq!(result.storage_class.as_deref(), Some(storage_class)); @@ -80,7 +150,7 @@ async fn test_head_object_404_key() { let client: S3CrtClient = get_test_client(); - let result = client.head_object(&bucket, &key).await; + let result = client.head_object(&bucket, &key, &HeadObjectParams::new()).await; assert!(matches!( result, Err(ObjectClientError::ServiceError(HeadObjectError::NotFound)) @@ -95,7 +165,9 @@ async fn test_head_object_404_bucket() { let client: S3CrtClient = get_test_client(); - let result = client.head_object("DOC-EXAMPLE-BUCKET", &key).await; + let result = client + .head_object("DOC-EXAMPLE-BUCKET", &key, &HeadObjectParams::new()) + .await; assert!(matches!( result, Err(ObjectClientError::ServiceError(HeadObjectError::NotFound)) @@ -111,7 +183,7 @@ async fn test_head_object_no_perm() { let client: S3CrtClient = get_test_client(); - let result = client.head_object(&bucket, &key).await; + let result = client.head_object(&bucket, &key, &HeadObjectParams::new()).await; assert!(matches!( result, Err(ObjectClientError::ClientError(S3RequestError::Forbidden(_, _))) @@ -138,7 +210,10 @@ async fn test_head_object_restored() { .unwrap(); let client: S3CrtClient = get_test_client(); - let result = client.head_object(&bucket, &key).await.expect("head_object failed"); + let result = client + .head_object(&bucket, &key, &HeadObjectParams::new()) + .await + .expect("head_object failed"); assert!( result.restore_status.is_none(), @@ -168,7 +243,10 @@ async fn test_head_object_restored() { let start = Instant::now(); let mut timeout_exceeded = true; while start.elapsed() < timeout { - let response = client.head_object(&bucket, &key).await.expect("head_object failed"); + let response = client + .head_object(&bucket, &key, &HeadObjectParams::new()) + .await + .expect("head_object failed"); if let Some(RestoreStatus::Restored { expiry: _ }) = response.restore_status { timeout_exceeded = false; break; diff --git a/mountpoint-s3-client/tests/metrics.rs b/mountpoint-s3-client/tests/metrics.rs index fdc53f48f..566b393a0 100644 --- a/mountpoint-s3-client/tests/metrics.rs +++ b/mountpoint-s3-client/tests/metrics.rs @@ -20,6 +20,7 @@ use metrics::{ Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, Recorder, SharedString, Unit, }; use mountpoint_s3_client::error::ObjectClientError; +use mountpoint_s3_client::types::HeadObjectParams; use mountpoint_s3_client::{ObjectClient, S3CrtClient, S3RequestError}; use regex::Regex; use rusty_fork::rusty_fork_test; @@ -233,7 +234,7 @@ async fn test_head_object_403() { let client: S3CrtClient = get_test_client(); let err = client - .head_object(&bucket, "some-key") + .head_object(&bucket, "some-key", &HeadObjectParams::new()) .await .expect_err("head to no-permissions bucket should fail"); assert!(matches!( diff --git a/mountpoint-s3-client/tests/network_interface_config.rs b/mountpoint-s3-client/tests/network_interface_config.rs index f310ff622..cc818df3d 100644 --- a/mountpoint-s3-client/tests/network_interface_config.rs +++ b/mountpoint-s3-client/tests/network_interface_config.rs @@ -7,6 +7,7 @@ use test_case::test_case; use common::*; use mountpoint_s3_client::config::{EndpointConfig, S3ClientConfig}; use mountpoint_s3_client::error::{HeadObjectError, ObjectClientError::ServiceError}; +use mountpoint_s3_client::types::HeadObjectParams; use mountpoint_s3_client::{ObjectClient, S3CrtClient}; #[tokio::test] @@ -21,7 +22,7 @@ async fn test_empty_list() { let client = S3CrtClient::new(config).expect("client should create OK"); let err = client - .head_object(&bucket, &key) + .head_object(&bucket, &key, &HeadObjectParams::new()) .await .expect_err("head_object should fail as the key doesn't exist"); assert!( @@ -43,7 +44,7 @@ async fn test_one_interface_ok() { let client = S3CrtClient::new(config).expect("client should create OK"); let err = client - .head_object(&bucket, &key) + .head_object(&bucket, &key, &HeadObjectParams::new()) .await .expect_err("head_object should fail as the key doesn't exist"); assert!( diff --git a/mountpoint-s3-client/tests/put_object.rs b/mountpoint-s3-client/tests/put_object.rs index 41cfc671b..d0c6032a9 100644 --- a/mountpoint-s3-client/tests/put_object.rs +++ b/mountpoint-s3-client/tests/put_object.rs @@ -11,7 +11,8 @@ use mountpoint_s3_client::checksums::crc32c_to_base64; use mountpoint_s3_client::config::{EndpointConfig, S3ClientConfig}; use mountpoint_s3_client::error::{GetObjectError, ObjectClientError}; use mountpoint_s3_client::types::{ - ChecksumAlgorithm, ObjectClientResult, PutObjectParams, PutObjectResult, PutObjectTrailingChecksums, + ChecksumAlgorithm, HeadObjectParams, ObjectClientResult, PutObjectParams, PutObjectResult, + PutObjectTrailingChecksums, }; use mountpoint_s3_client::{ObjectClient, PutObjectRequest, S3CrtClient, S3RequestError}; use mountpoint_s3_crt::checksums::crc32c; @@ -660,7 +661,7 @@ async fn test_concurrent_put_objects(throughput_target_gbps: f64, max_concurrent // Also try to issue an unrelated request (head_object). tokio::time::timeout(TIMEOUT, async { client - .head_object(&bucket, ¬_existing_key) + .head_object(&bucket, ¬_existing_key, &HeadObjectParams::new()) .await .expect_err("head object should fail") }) diff --git a/mountpoint-s3/examples/prefetch_benchmark.rs b/mountpoint-s3/examples/prefetch_benchmark.rs index 3bdb76fe5..620303b83 100644 --- a/mountpoint-s3/examples/prefetch_benchmark.rs +++ b/mountpoint-s3/examples/prefetch_benchmark.rs @@ -9,6 +9,7 @@ use mountpoint_s3::mem_limiter::MemoryLimiter; use mountpoint_s3::object::ObjectId; use mountpoint_s3::prefetch::{default_prefetch, Prefetch, PrefetchResult}; use mountpoint_s3_client::config::{EndpointConfig, S3ClientConfig}; +use mountpoint_s3_client::types::HeadObjectParams; use mountpoint_s3_client::{ObjectClient, S3CrtClient}; use mountpoint_s3_crt::common::rust_log_adapter::RustLogAdapter; use sysinfo::{RefreshKind, System}; @@ -110,7 +111,8 @@ fn main() { }; let mem_limiter = Arc::new(MemoryLimiter::new(client.clone(), max_memory_target)); - let head_object_result = block_on(client.head_object(bucket, key)).expect("HeadObject failed"); + let head_object_result = + block_on(client.head_object(bucket, key, &HeadObjectParams::new())).expect("HeadObject failed"); let size = head_object_result.size; let etag = head_object_result.etag; diff --git a/mountpoint-s3/src/superblock.rs b/mountpoint-s3/src/superblock.rs index 5a2cba7c6..c63d61f63 100644 --- a/mountpoint-s3/src/superblock.rs +++ b/mountpoint-s3/src/superblock.rs @@ -30,7 +30,7 @@ use anyhow::anyhow; use futures::{select_biased, FutureExt}; use mountpoint_s3_client::error::{HeadObjectError, ObjectClientError}; use mountpoint_s3_client::error_metadata::ProvideErrorMetadata; -use mountpoint_s3_client::types::HeadObjectResult; +use mountpoint_s3_client::types::{HeadObjectParams, HeadObjectResult}; use mountpoint_s3_client::ObjectClient; use thiserror::Error; use time::OffsetDateTime; @@ -688,7 +688,8 @@ impl SuperblockInner { // "/" to the prefix in the request, the first common prefix we'll get back will be // "dir-1/", because that precedes "dir/" in lexicographic order. Doing the // ListObjects with "/" appended makes sure we always observe the correct prefix. - let mut file_lookup = client.head_object(&self.bucket, &full_path).fuse(); + let head_object_params = HeadObjectParams::new(); + let mut file_lookup = client.head_object(&self.bucket, &full_path, &head_object_params).fuse(); let mut dir_lookup = client .list_objects(&self.bucket, None, "/", 1, &full_path_suffixed) .fuse(); @@ -1273,7 +1274,7 @@ mod tests { .expect("inode should exist"); // Grab last modified time according to mock S3 let modified_time = client - .head_object(bucket, file.inode.full_key()) + .head_object(bucket, file.inode.full_key(), &HeadObjectParams::new()) .await .expect("object should exist") .last_modified; diff --git a/mountpoint-s3/tests/reftests/reference.rs b/mountpoint-s3/tests/reftests/reference.rs index 87d7c51fb..b697da2bc 100644 --- a/mountpoint-s3/tests/reftests/reference.rs +++ b/mountpoint-s3/tests/reftests/reference.rs @@ -8,7 +8,7 @@ use tracing::trace; #[derive(Debug)] pub enum File { Local, - Remote(MockObject), + Remote(Box), } #[derive(Debug)] @@ -306,7 +306,7 @@ fn build_reference<'a>(flat: impl Iterator) #[derive(Debug)] enum RefNode { Directory(Rc>>), - File(MockObject), + File(Box), } impl RefNode { @@ -356,7 +356,7 @@ fn build_reference<'a>(flat: impl Iterator) if valid_inode_name(file_name) && should_create { leaf_dir .borrow_mut() - .insert(file_name.to_string(), RefNode::File(file.clone())); + .insert(file_name.to_string(), RefNode::File(Box::new(file.clone()))); } }