From 3c371f3088d17cc8e35a06dcf0915416c9d067e1 Mon Sep 17 00:00:00 2001 From: Burak Date: Wed, 4 Sep 2024 18:03:50 +0100 Subject: [PATCH] Update CRT submodules to latest (#997) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update mountpoint-s3-crt-sys crate excludes to reduce package size Signed-off-by: Daniel Carl Jones Signed-off-by: Burak Varlı * Update CRT submodules to latest releases Signed-off-by: Daniel Carl Jones Signed-off-by: Burak Varlı * WIP: Add testing for https://github.com/awslabs/mountpoint-s3/issues/927 Signed-off-by: Burak Varlı * Gate scoped credential test Signed-off-by: Burak Varlı * Update `test_credential_process_behind_source_profile` to use role Signed-off-by: Burak Varlı * Remove unused import Signed-off-by: Burak Varlı * Remove TODOs from CHANGELOG Signed-off-by: Burak Varlı * Update CRT submodules to latest Signed-off-by: Burak Varlı * Remove feature gate from `test_credential_process_behind_source_profile` Signed-off-by: Burak Varlı * Fix Clippy failures Signed-off-by: Burak Varlı * More Clippy fixes Signed-off-by: Burak Varlı * Fix formatting Signed-off-by: Burak Varlı * Update test failure message Co-authored-by: Daniel Carl Jones Signed-off-by: Burak --------- Signed-off-by: Daniel Carl Jones Signed-off-by: Burak Varlı Signed-off-by: Burak Co-authored-by: Daniel Carl Jones Co-authored-by: Daniel Carl Jones --- mountpoint-s3-client/tests/auth.rs | 133 +++++++++++++++++++- mountpoint-s3-crt-sys/Cargo.toml | 4 +- mountpoint-s3-crt-sys/crt/aws-c-auth | 2 +- mountpoint-s3-crt-sys/crt/aws-c-cal | 2 +- mountpoint-s3-crt-sys/crt/aws-c-common | 2 +- mountpoint-s3-crt-sys/crt/aws-c-compression | 2 +- mountpoint-s3-crt-sys/crt/aws-c-http | 2 +- mountpoint-s3-crt-sys/crt/aws-c-io | 2 +- mountpoint-s3-crt-sys/crt/aws-c-sdkutils | 2 +- mountpoint-s3-crt-sys/crt/aws-lc | 2 +- mountpoint-s3-crt-sys/crt/s2n-tls | 2 +- mountpoint-s3/CHANGELOG.md | 7 ++ 12 files changed, 151 insertions(+), 11 deletions(-) diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index d92fcf482..857bb0d62 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -4,6 +4,7 @@ pub mod common; use std::io::Write; use std::option::Option::None; +use std::writeln; use aws_sdk_s3::primitives::ByteStream; use bytes::Bytes; @@ -13,7 +14,6 @@ use common::creds::{get_sdk_default_chain_creds, get_subsession_iam_role}; use common::*; use futures::StreamExt; use mountpoint_s3_client::config::{EndpointConfig, S3ClientAuthConfig, S3ClientConfig}; -#[cfg(not(feature = "s3express_tests"))] use mountpoint_s3_client::error::ObjectClientError; #[cfg(not(feature = "s3express_tests"))] use mountpoint_s3_client::S3RequestError; @@ -240,6 +240,130 @@ async fn test_profile_provider_assume_role_async() { check_get_result(result, None, &body[..]).await; } +async fn test_credential_process_behind_source_profile_async() { + let (bucket, prefix) = get_test_bucket_and_prefix("test_credential_process_behind_source_profile"); + + // Create a test file in "{prefix}/hello" + { + let sdk_client = get_test_sdk_client().await; + let key = format!("{prefix}/hello"); + let body = b"hello world!"; + sdk_client + .put_object() + .bucket(&bucket) + .key(&key) + .body(ByteStream::from(Bytes::from_static(body))) + .send() + .await + .unwrap(); + } + + // Get some static credentials by just using the SDK's default provider, which we know works. + let credentials = get_sdk_default_chain_creds().await; + + // Create two credential files to be used in `credential_process`, + // one with correct credentials and one with incorrect credentials. + let (correct_credential_file, incorrect_credential_file) = { + let mut correct = NamedTempFile::new().unwrap(); + let mut incorrect = NamedTempFile::new().unwrap(); + let json_response = r#"{ + "Version": 1, + "AccessKeyId": "__AWS_ACCESS_KEY_ID__", + "SecretAccessKey": "__AWS_SECRET_ACCESS_KEY__", + "SessionToken": "__AWS_SESSION_TOKEN__", + "Expiration": "2099-08-20T00:05:35+00:00" + }"#; + + correct + .write_all( + json_response + .replace("__AWS_ACCESS_KEY_ID__", credentials.access_key_id()) + .replace("__AWS_SECRET_ACCESS_KEY__", credentials.secret_access_key()) + .replace("__AWS_SESSION_TOKEN__", credentials.session_token().unwrap()) + .as_bytes(), + ) + .unwrap(); + + incorrect + .write_all( + json_response + .replace("__AWS_ACCESS_KEY_ID__", &credentials.access_key_id()[..10]) + .replace("__AWS_SECRET_ACCESS_KEY__", credentials.secret_access_key()) + .replace("__AWS_SESSION_TOKEN__", credentials.session_token().unwrap()) + .as_bytes(), + ) + .unwrap(); + + (correct, incorrect) + }; + + let mut config_file = NamedTempFile::new().unwrap(); + + // Create two source profiles to provide credentials from previously created files using `credential_process`. + let (correct_source_profile, incorrect_source_profile) = { + let correct = "correct-source-profile"; + writeln!(config_file, "[profile {}]", correct).unwrap(); + writeln!( + config_file, + "credential_process=cat {}", + correct_credential_file.path().to_string_lossy() + ) + .unwrap(); + let incorrect = "incorrect-source-profile"; + writeln!(config_file, "[profile {}]", incorrect).unwrap(); + writeln!( + config_file, + "credential_process=cat {}", + incorrect_credential_file.path().to_string_lossy() + ) + .unwrap(); + (correct, incorrect) + }; + + // Create two profiles to assume our test role with previously created source profiles. + let (correct_profile, incorrect_profile) = { + let correct = "correct-profile"; + writeln!(config_file, "[profile {}]", correct).unwrap(); + writeln!(config_file, "role_arn={}", get_subsession_iam_role()).unwrap(); + writeln!(config_file, "source_profile={}", correct_source_profile).unwrap(); + writeln!(config_file, "region={}", &get_test_region()).unwrap(); + let incorrect = "incorrect-profile"; + writeln!(config_file, "[profile {}]", incorrect).unwrap(); + writeln!(config_file, "role_arn={}", get_subsession_iam_role()).unwrap(); + writeln!(config_file, "source_profile={}", incorrect_source_profile).unwrap(); + writeln!(config_file, "region={}", &get_test_region()).unwrap(); + (correct, incorrect) + }; + + config_file.flush().unwrap(); + + // Set up the environment variables to use this new config file. This is only OK to do because + // this test is run in a forked process, so won't affect any other concurrently running tests. + std::env::set_var("AWS_CONFIG_FILE", config_file.path().as_os_str()); + + // With correct profile, things should be fine + let config = S3ClientConfig::new() + .auth_config(S3ClientAuthConfig::Profile(correct_profile.to_owned())) + .endpoint_config(EndpointConfig::new(&get_test_region())); + let client = S3CrtClient::new(config).unwrap(); + let _result = client + .list_objects(&bucket, None, "/", 10, &format!("{prefix}foo/")) + .await + .expect("list_objects should succeed"); + + // With incorrect profile, requests should fail with a client error + let config = S3ClientConfig::new() + .auth_config(S3ClientAuthConfig::Profile(incorrect_profile.to_owned())) + .endpoint_config(EndpointConfig::new(&get_test_region())); + let client = S3CrtClient::new(config).unwrap(); + let err = client + .list_objects(&bucket, None, "/", 10, &format!("{prefix}/")) + .await + .expect_err("should fail when using invalid credentials"); + assert!(matches!(err, ObjectClientError::ClientError(_))); + drop(config_file); +} + rusty_fork_test! { #[test] fn test_profile_provider_static() { @@ -254,6 +378,13 @@ rusty_fork_test! { let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); runtime.block_on(test_profile_provider_assume_role_async()); } + + #[test] + fn test_credential_process_behind_source_profile() { + // rusty_fork doesn't support async tests, so build an SDK-usable runtime manually + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + runtime.block_on(test_credential_process_behind_source_profile_async()); + } } /// Test using a client with scoped-down credentials diff --git a/mountpoint-s3-crt-sys/Cargo.toml b/mountpoint-s3-crt-sys/Cargo.toml index c85af4a32..1dee338c1 100644 --- a/mountpoint-s3-crt-sys/Cargo.toml +++ b/mountpoint-s3-crt-sys/Cargo.toml @@ -62,7 +62,9 @@ exclude = [ ] [build-dependencies] -bindgen = { version = "0.66.1", default-features = false, features = ["runtime"] } +bindgen = { version = "0.66.1", default-features = false, features = [ + "runtime", +] } cc = "1.0.73" cmake = "0.1.48" rustflags = "0.1.1" diff --git a/mountpoint-s3-crt-sys/crt/aws-c-auth b/mountpoint-s3-crt-sys/crt/aws-c-auth index d78952529..877c029fc 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-auth +++ b/mountpoint-s3-crt-sys/crt/aws-c-auth @@ -1 +1 @@ -Subproject commit d7895252915259d037f10be223369eb99e837471 +Subproject commit 877c029fc4e93d205f9c6855188c3c51f6b497b4 diff --git a/mountpoint-s3-crt-sys/crt/aws-c-cal b/mountpoint-s3-crt-sys/crt/aws-c-cal index bc0d71b66..2cb1d2eac 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-cal +++ b/mountpoint-s3-crt-sys/crt/aws-c-cal @@ -1 +1 @@ -Subproject commit bc0d71b66deea9e1a2d3a073c9f1ece9286b9e60 +Subproject commit 2cb1d2eac925e2dbc45025eb89af82bd790c23a0 diff --git a/mountpoint-s3-crt-sys/crt/aws-c-common b/mountpoint-s3-crt-sys/crt/aws-c-common index 67601bbbc..672cc0032 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-common +++ b/mountpoint-s3-crt-sys/crt/aws-c-common @@ -1 +1 @@ -Subproject commit 67601bbbce6ea1c26191f235afc50007a4e796c5 +Subproject commit 672cc0032eb28d69fbdd22c9463253c89d7a6f30 diff --git a/mountpoint-s3-crt-sys/crt/aws-c-compression b/mountpoint-s3-crt-sys/crt/aws-c-compression index ea1d421a4..f36d01672 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-compression +++ b/mountpoint-s3-crt-sys/crt/aws-c-compression @@ -1 +1 @@ -Subproject commit ea1d421a421ad83a540309a94c38d50b6a5d836b +Subproject commit f36d01672d61e49d96a777870d456f66fa391cd4 diff --git a/mountpoint-s3-crt-sys/crt/aws-c-http b/mountpoint-s3-crt-sys/crt/aws-c-http index a2fb16c43..4e74ab1e3 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-http +++ b/mountpoint-s3-crt-sys/crt/aws-c-http @@ -1 +1 @@ -Subproject commit a2fb16c431152bd677093f9910f405a37533ad5a +Subproject commit 4e74ab1e3702763e0b87bd1752f5a37c2f0400ac diff --git a/mountpoint-s3-crt-sys/crt/aws-c-io b/mountpoint-s3-crt-sys/crt/aws-c-io index e5fe40e11..c345d7727 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-io +++ b/mountpoint-s3-crt-sys/crt/aws-c-io @@ -1 +1 @@ -Subproject commit e5fe40e11d97221c25b3b40ad2949822b3422993 +Subproject commit c345d77274db83c0c2e30331814093e7c84c45e2 diff --git a/mountpoint-s3-crt-sys/crt/aws-c-sdkutils b/mountpoint-s3-crt-sys/crt/aws-c-sdkutils index 8c7af71f9..4658412a6 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-sdkutils +++ b/mountpoint-s3-crt-sys/crt/aws-c-sdkutils @@ -1 +1 @@ -Subproject commit 8c7af71f91ed5b9d2a043d51f120495f43723f80 +Subproject commit 4658412a61ad5749db92a8d1e0717cb5e76ada1c diff --git a/mountpoint-s3-crt-sys/crt/aws-lc b/mountpoint-s3-crt-sys/crt/aws-lc index 47333e181..2f1879759 160000 --- a/mountpoint-s3-crt-sys/crt/aws-lc +++ b/mountpoint-s3-crt-sys/crt/aws-lc @@ -1 +1 @@ -Subproject commit 47333e18117875148fc737c38c2d5586b45c7dfc +Subproject commit 2f1879759b2e0fc70592665bdf10087b64f44b7d diff --git a/mountpoint-s3-crt-sys/crt/s2n-tls b/mountpoint-s3-crt-sys/crt/s2n-tls index 138e3ece9..87f4a0585 160000 --- a/mountpoint-s3-crt-sys/crt/s2n-tls +++ b/mountpoint-s3-crt-sys/crt/s2n-tls @@ -1 +1 @@ -Subproject commit 138e3ece9e457c2f824a85b63095737f30d624a9 +Subproject commit 87f4a0585dc3056433f193b9305f587cff239be3 diff --git a/mountpoint-s3/CHANGELOG.md b/mountpoint-s3/CHANGELOG.md index 28b370b51..32f368c69 100644 --- a/mountpoint-s3/CHANGELOG.md +++ b/mountpoint-s3/CHANGELOG.md @@ -1,3 +1,10 @@ +## Unreleased + +### Other changes + +* Fix an issue where `credential_process` field would not be picked up correctly when using `source_profile`. ([awslabs/aws-c-auth#245](https://github.com/awslabs/aws-c-auth/pull/245)) +* Fix an issue where `credential_process` field would not be picked up correctly when using `--profile `. ([awslabs/aws-c-auth#245](https://github.com/awslabs/aws-c-auth/pull/245)) + ## v1.8.0 ### New features